2024-11-15 22:15:20,163 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 22:15:20,176 main DEBUG Took 0.010806 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 22:15:20,176 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 22:15:20,177 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 22:15:20,178 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 22:15:20,179 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:15:20,187 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 22:15:20,202 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:15:20,204 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:15:20,205 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:15:20,206 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:15:20,207 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:15:20,207 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:15:20,209 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:15:20,210 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:15:20,211 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:15:20,211 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:15:20,213 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:15:20,214 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:15:20,214 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:15:20,215 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:15:20,216 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:15:20,216 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:15:20,217 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:15:20,217 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:15:20,218 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:15:20,219 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:15:20,219 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:15:20,220 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:15:20,221 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:15:20,221 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:15:20,222 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:15:20,222 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 22:15:20,224 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:15:20,226 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 22:15:20,229 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 22:15:20,230 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 22:15:20,232 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 22:15:20,232 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 22:15:20,240 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 22:15:20,243 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 22:15:20,245 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 22:15:20,245 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 22:15:20,246 main DEBUG createAppenders(={Console}) 2024-11-15 22:15:20,247 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-15 22:15:20,247 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 22:15:20,247 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-15 22:15:20,248 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 22:15:20,248 main DEBUG OutputStream closed 2024-11-15 22:15:20,248 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 22:15:20,248 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 22:15:20,249 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-15 22:15:20,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 22:15:20,342 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 22:15:20,343 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 22:15:20,345 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 22:15:20,345 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 22:15:20,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 22:15:20,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 22:15:20,347 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 22:15:20,347 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 22:15:20,348 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 22:15:20,348 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 22:15:20,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 22:15:20,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 22:15:20,350 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 22:15:20,350 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 22:15:20,350 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 22:15:20,351 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 22:15:20,352 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 22:15:20,355 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 22:15:20,356 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-15 22:15:20,356 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 22:15:20,357 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-15T22:15:20,617 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e 2024-11-15 22:15:20,619 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 22:15:20,620 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T22:15:20,627 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-15T22:15:20,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T22:15:20,662 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/cluster_a6235007-a423-ea83-cf16-0fd92ac68712, deleteOnExit=true 2024-11-15T22:15:20,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T22:15:20,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/test.cache.data in system properties and HBase conf 2024-11-15T22:15:20,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T22:15:20,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/hadoop.log.dir in system properties and HBase conf 2024-11-15T22:15:20,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T22:15:20,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T22:15:20,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T22:15:20,751 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T22:15:20,837 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T22:15:20,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T22:15:20,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T22:15:20,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T22:15:20,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T22:15:20,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T22:15:20,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T22:15:20,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T22:15:20,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T22:15:20,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T22:15:20,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/nfs.dump.dir in system properties and HBase conf 2024-11-15T22:15:20,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/java.io.tmpdir in system properties and HBase conf 2024-11-15T22:15:20,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T22:15:20,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T22:15:20,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T22:15:21,978 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T22:15:22,053 INFO [Time-limited test {}] log.Log(170): Logging initialized @2657ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T22:15:22,123 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T22:15:22,187 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T22:15:22,224 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T22:15:22,224 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T22:15:22,226 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T22:15:22,239 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T22:15:22,242 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/hadoop.log.dir/,AVAILABLE} 2024-11-15T22:15:22,243 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T22:15:22,449 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/java.io.tmpdir/jetty-localhost-40045-hadoop-hdfs-3_4_1-tests_jar-_-any-6022245723484722323/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T22:15:22,456 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:40045} 2024-11-15T22:15:22,457 INFO [Time-limited test {}] server.Server(415): Started @3061ms 2024-11-15T22:15:23,049 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T22:15:23,059 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T22:15:23,061 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T22:15:23,061 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T22:15:23,061 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T22:15:23,063 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6aa6ac55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/hadoop.log.dir/,AVAILABLE} 2024-11-15T22:15:23,064 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58f8c8ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T22:15:23,178 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d1d0948{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/java.io.tmpdir/jetty-localhost-43613-hadoop-hdfs-3_4_1-tests_jar-_-any-1203801096104656422/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T22:15:23,178 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@49b49063{HTTP/1.1, (http/1.1)}{localhost:43613} 2024-11-15T22:15:23,179 INFO [Time-limited test {}] server.Server(415): Started @3783ms 2024-11-15T22:15:23,229 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T22:15:24,467 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/cluster_a6235007-a423-ea83-cf16-0fd92ac68712/data/data1/current/BP-14505792-172.17.0.3-1731708921465/current, will proceed with Du for space computation calculation, 2024-11-15T22:15:24,467 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/cluster_a6235007-a423-ea83-cf16-0fd92ac68712/data/data2/current/BP-14505792-172.17.0.3-1731708921465/current, will proceed with Du for space computation calculation, 2024-11-15T22:15:24,517 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T22:15:24,597 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x12120278f9a93993 with lease ID 0xe778e01cdc13111f: Processing first storage report for DS-fbaa7e2b-1311-447c-a7c5-ccdf11aa0038 from datanode DatanodeRegistration(127.0.0.1:38455, datanodeUuid=2f720da5-324f-4b5d-90f7-54311f9c5ee1, infoPort=40791, infoSecurePort=0, ipcPort=38445, storageInfo=lv=-57;cid=testClusterID;nsid=142778691;c=1731708921465) 2024-11-15T22:15:24,600 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x12120278f9a93993 with lease ID 0xe778e01cdc13111f: from storage DS-fbaa7e2b-1311-447c-a7c5-ccdf11aa0038 node DatanodeRegistration(127.0.0.1:38455, datanodeUuid=2f720da5-324f-4b5d-90f7-54311f9c5ee1, infoPort=40791, infoSecurePort=0, ipcPort=38445, storageInfo=lv=-57;cid=testClusterID;nsid=142778691;c=1731708921465), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-15T22:15:24,601 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x12120278f9a93993 with lease ID 0xe778e01cdc13111f: Processing first storage report for DS-b411dac1-eee8-46d3-8f50-422d73a944f2 from datanode DatanodeRegistration(127.0.0.1:38455, datanodeUuid=2f720da5-324f-4b5d-90f7-54311f9c5ee1, infoPort=40791, infoSecurePort=0, ipcPort=38445, storageInfo=lv=-57;cid=testClusterID;nsid=142778691;c=1731708921465) 2024-11-15T22:15:24,601 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x12120278f9a93993 with lease ID 0xe778e01cdc13111f: from storage DS-b411dac1-eee8-46d3-8f50-422d73a944f2 node DatanodeRegistration(127.0.0.1:38455, datanodeUuid=2f720da5-324f-4b5d-90f7-54311f9c5ee1, infoPort=40791, infoSecurePort=0, ipcPort=38445, storageInfo=lv=-57;cid=testClusterID;nsid=142778691;c=1731708921465), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T22:15:24,650 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e 2024-11-15T22:15:24,935 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/cluster_a6235007-a423-ea83-cf16-0fd92ac68712/zookeeper_0, clientPort=51187, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/cluster_a6235007-a423-ea83-cf16-0fd92ac68712/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/cluster_a6235007-a423-ea83-cf16-0fd92ac68712/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T22:15:24,963 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51187 2024-11-15T22:15:24,976 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:15:24,979 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:15:25,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741825_1001 (size=7) 2024-11-15T22:15:25,394 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e with version=8 2024-11-15T22:15:25,394 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/hbase-staging 2024-11-15T22:15:25,578 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T22:15:26,016 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e611192d6313:0 server-side Connection retries=45 2024-11-15T22:15:26,030 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:15:26,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T22:15:26,056 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T22:15:26,056 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:15:26,057 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T22:15:26,394 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T22:15:26,489 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T22:15:26,506 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T22:15:26,512 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T22:15:26,545 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 6727 (auto-detected) 2024-11-15T22:15:26,547 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-15T22:15:26,578 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34969 2024-11-15T22:15:26,618 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34969 connecting to ZooKeeper ensemble=127.0.0.1:51187 2024-11-15T22:15:26,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:349690x0, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T22:15:26,817 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34969-0x10140912f520000 connected 2024-11-15T22:15:26,915 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:15:26,917 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:15:26,927 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T22:15:26,931 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e, hbase.cluster.distributed=false 2024-11-15T22:15:26,957 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T22:15:26,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34969 2024-11-15T22:15:26,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34969 2024-11-15T22:15:26,965 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34969 2024-11-15T22:15:26,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34969 2024-11-15T22:15:26,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34969 2024-11-15T22:15:27,065 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e611192d6313:0 server-side Connection retries=45 2024-11-15T22:15:27,066 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:15:27,066 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T22:15:27,067 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T22:15:27,067 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:15:27,067 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T22:15:27,069 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T22:15:27,072 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T22:15:27,073 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38779 2024-11-15T22:15:27,075 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38779 connecting to ZooKeeper ensemble=127.0.0.1:51187 2024-11-15T22:15:27,076 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:15:27,080 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:15:27,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387790x0, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T22:15:27,093 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387790x0, quorum=127.0.0.1:51187, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T22:15:27,093 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38779-0x10140912f520001 connected 2024-11-15T22:15:27,097 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T22:15:27,104 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T22:15:27,107 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T22:15:27,112 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T22:15:27,113 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38779 2024-11-15T22:15:27,113 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38779 2024-11-15T22:15:27,113 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38779 2024-11-15T22:15:27,114 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38779 2024-11-15T22:15:27,114 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38779 2024-11-15T22:15:27,134 DEBUG [M:0;e611192d6313:34969 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e611192d6313:34969 2024-11-15T22:15:27,136 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e611192d6313,34969,1731708925722 2024-11-15T22:15:27,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:15:27,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:15:27,155 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e611192d6313,34969,1731708925722 2024-11-15T22:15:27,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:15:27,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T22:15:27,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:15:27,188 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T22:15:27,189 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e611192d6313,34969,1731708925722 from backup master directory 2024-11-15T22:15:27,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:15:27,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e611192d6313,34969,1731708925722 2024-11-15T22:15:27,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:15:27,198 WARN [master/e611192d6313:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T22:15:27,199 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e611192d6313,34969,1731708925722 2024-11-15T22:15:27,201 INFO [master/e611192d6313:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T22:15:27,203 INFO [master/e611192d6313:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T22:15:27,256 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/hbase.id] with ID: 6325f831-4157-4d7d-bab4-cef03871eef5 2024-11-15T22:15:27,256 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/.tmp/hbase.id 2024-11-15T22:15:27,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741826_1002 (size=42) 2024-11-15T22:15:27,672 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/.tmp/hbase.id]:[hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/hbase.id] 2024-11-15T22:15:27,724 INFO [master/e611192d6313:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:15:27,729 INFO [master/e611192d6313:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T22:15:27,749 INFO [master/e611192d6313:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-15T22:15:27,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:15:27,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:15:27,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741827_1003 (size=196) 2024-11-15T22:15:27,842 INFO [master/e611192d6313:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:15:27,844 INFO [master/e611192d6313:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T22:15:27,857 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:27,861 INFO [master/e611192d6313:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T22:15:27,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741828_1004 (size=1189) 2024-11-15T22:15:28,314 INFO [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store 2024-11-15T22:15:28,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741829_1005 (size=34) 2024-11-15T22:15:28,744 INFO [master/e611192d6313:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T22:15:28,747 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:15:28,748 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T22:15:28,749 INFO [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:15:28,749 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:15:28,751 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T22:15:28,751 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:15:28,751 INFO [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:15:28,753 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731708928748Disabling compacts and flushes for region at 1731708928748Disabling writes for close at 1731708928751 (+3 ms)Writing region close event to WAL at 1731708928751Closed at 1731708928751 2024-11-15T22:15:28,755 WARN [master/e611192d6313:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/.initializing 2024-11-15T22:15:28,755 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/WALs/e611192d6313,34969,1731708925722 2024-11-15T22:15:28,763 INFO [master/e611192d6313:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T22:15:28,779 INFO [master/e611192d6313:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e611192d6313%2C34969%2C1731708925722, suffix=, logDir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/WALs/e611192d6313,34969,1731708925722, archiveDir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/oldWALs, maxLogs=10 2024-11-15T22:15:28,806 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/WALs/e611192d6313,34969,1731708925722/e611192d6313%2C34969%2C1731708925722.1731708928784, exclude list is [], retry=0 2024-11-15T22:15:28,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38455,DS-fbaa7e2b-1311-447c-a7c5-ccdf11aa0038,DISK] 2024-11-15T22:15:28,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T22:15:28,861 INFO [master/e611192d6313:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/WALs/e611192d6313,34969,1731708925722/e611192d6313%2C34969%2C1731708925722.1731708928784 2024-11-15T22:15:28,862 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40791:40791)] 2024-11-15T22:15:28,862 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:15:28,863 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:15:28,866 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:15:28,867 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:15:28,912 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:15:28,947 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T22:15:28,952 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:28,958 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:15:28,959 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:15:28,963 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T22:15:28,963 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:28,964 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:15:28,964 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:15:28,968 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T22:15:28,968 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:28,969 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:15:28,969 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:15:28,972 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T22:15:28,973 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:28,974 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:15:28,974 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:15:28,979 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:15:28,980 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:15:28,985 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:15:28,986 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:15:28,991 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T22:15:28,996 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:15:29,006 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:15:29,008 INFO [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68897808, jitterRate=0.0266573429107666}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T22:15:29,018 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731708928879Initializing all the Stores at 1731708928881 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708928882 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708928883 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708928883Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708928883Cleaning up temporary data from old regions at 1731708928987 (+104 ms)Region opened successfully at 1731708929018 (+31 ms) 2024-11-15T22:15:29,023 INFO [master/e611192d6313:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T22:15:29,069 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5774c48e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e611192d6313/172.17.0.3:0 2024-11-15T22:15:29,109 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T22:15:29,121 INFO [master/e611192d6313:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T22:15:29,121 INFO [master/e611192d6313:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T22:15:29,124 INFO [master/e611192d6313:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:15:29,126 INFO [master/e611192d6313:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T22:15:29,130 INFO [master/e611192d6313:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T22:15:29,131 INFO [master/e611192d6313:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T22:15:29,157 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T22:15:29,166 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T22:15:29,239 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T22:15:29,242 INFO [master/e611192d6313:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T22:15:29,244 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T22:15:29,250 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T22:15:29,252 INFO [master/e611192d6313:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T22:15:29,256 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T22:15:29,268 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T22:15:29,269 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T22:15:29,281 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T22:15:29,300 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T22:15:29,310 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T22:15:29,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T22:15:29,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T22:15:29,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:15:29,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:15:29,329 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e611192d6313,34969,1731708925722, sessionid=0x10140912f520000, setting cluster-up flag (Was=false) 2024-11-15T22:15:29,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:15:29,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:15:29,387 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T22:15:29,391 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e611192d6313,34969,1731708925722 2024-11-15T22:15:29,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:15:29,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:15:29,450 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T22:15:29,455 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e611192d6313,34969,1731708925722 2024-11-15T22:15:29,464 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T22:15:29,520 INFO [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(746): ClusterId : 6325f831-4157-4d7d-bab4-cef03871eef5 2024-11-15T22:15:29,523 DEBUG [RS:0;e611192d6313:38779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T22:15:29,530 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:15:29,540 INFO [master/e611192d6313:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T22:15:29,546 INFO [master/e611192d6313:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T22:15:29,554 DEBUG [RS:0;e611192d6313:38779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T22:15:29,554 DEBUG [RS:0;e611192d6313:38779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T22:15:29,552 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e611192d6313,34969,1731708925722 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T22:15:29,567 DEBUG [RS:0;e611192d6313:38779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T22:15:29,568 DEBUG [RS:0;e611192d6313:38779 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e79446, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e611192d6313/172.17.0.3:0 2024-11-15T22:15:29,569 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e611192d6313:0, corePoolSize=5, maxPoolSize=5 2024-11-15T22:15:29,569 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e611192d6313:0, corePoolSize=5, maxPoolSize=5 2024-11-15T22:15:29,569 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e611192d6313:0, corePoolSize=5, maxPoolSize=5 2024-11-15T22:15:29,569 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e611192d6313:0, corePoolSize=5, maxPoolSize=5 2024-11-15T22:15:29,569 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e611192d6313:0, corePoolSize=10, maxPoolSize=10 2024-11-15T22:15:29,570 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:15:29,570 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e611192d6313:0, corePoolSize=2, maxPoolSize=2 2024-11-15T22:15:29,570 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:15:29,571 INFO [master/e611192d6313:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708959571 2024-11-15T22:15:29,573 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T22:15:29,574 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T22:15:29,575 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T22:15:29,576 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T22:15:29,579 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T22:15:29,579 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T22:15:29,580 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T22:15:29,580 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T22:15:29,582 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:29,583 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T22:15:29,583 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:29,586 DEBUG [RS:0;e611192d6313:38779 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e611192d6313:38779 2024-11-15T22:15:29,587 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T22:15:29,588 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T22:15:29,589 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T22:15:29,589 INFO [RS:0;e611192d6313:38779 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T22:15:29,590 INFO [RS:0;e611192d6313:38779 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T22:15:29,590 DEBUG [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T22:15:29,591 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T22:15:29,591 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T22:15:29,593 INFO [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(2659): reportForDuty to master=e611192d6313,34969,1731708925722 with port=38779, startcode=1731708927034 2024-11-15T22:15:29,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741831_1007 (size=1321) 2024-11-15T22:15:29,606 DEBUG [RS:0;e611192d6313:38779 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T22:15:29,608 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e611192d6313:0:becomeActiveMaster-HFileCleaner.large.0-1731708929592,5,FailOnTimeoutGroup] 2024-11-15T22:15:29,609 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e611192d6313:0:becomeActiveMaster-HFileCleaner.small.0-1731708929608,5,FailOnTimeoutGroup] 2024-11-15T22:15:29,609 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:29,609 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T22:15:29,610 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:29,611 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:29,638 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48211, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T22:15:29,644 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34969 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e611192d6313,38779,1731708927034 2024-11-15T22:15:29,646 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34969 {}] master.ServerManager(517): Registering regionserver=e611192d6313,38779,1731708927034 2024-11-15T22:15:29,659 DEBUG [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e 2024-11-15T22:15:29,660 DEBUG [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36935 2024-11-15T22:15:29,660 DEBUG [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T22:15:29,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T22:15:29,672 DEBUG [RS:0;e611192d6313:38779 {}] zookeeper.ZKUtil(111): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e611192d6313,38779,1731708927034 2024-11-15T22:15:29,672 WARN [RS:0;e611192d6313:38779 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T22:15:29,672 INFO [RS:0;e611192d6313:38779 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T22:15:29,672 DEBUG [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/WALs/e611192d6313,38779,1731708927034 2024-11-15T22:15:29,674 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e611192d6313,38779,1731708927034] 2024-11-15T22:15:29,695 INFO [RS:0;e611192d6313:38779 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T22:15:29,710 INFO [RS:0;e611192d6313:38779 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T22:15:29,715 INFO [RS:0;e611192d6313:38779 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T22:15:29,716 INFO [RS:0;e611192d6313:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:29,717 INFO [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T22:15:29,722 INFO [RS:0;e611192d6313:38779 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T22:15:29,723 INFO [RS:0;e611192d6313:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:29,723 DEBUG [RS:0;e611192d6313:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:15:29,723 DEBUG [RS:0;e611192d6313:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:15:29,723 DEBUG [RS:0;e611192d6313:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:15:29,724 DEBUG [RS:0;e611192d6313:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:15:29,724 DEBUG [RS:0;e611192d6313:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:15:29,724 DEBUG [RS:0;e611192d6313:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0, corePoolSize=2, maxPoolSize=2 2024-11-15T22:15:29,724 DEBUG [RS:0;e611192d6313:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:15:29,724 DEBUG [RS:0;e611192d6313:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:15:29,725 DEBUG [RS:0;e611192d6313:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:15:29,725 DEBUG [RS:0;e611192d6313:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:15:29,725 DEBUG [RS:0;e611192d6313:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:15:29,725 DEBUG [RS:0;e611192d6313:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:15:29,726 DEBUG [RS:0;e611192d6313:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e611192d6313:0, corePoolSize=3, maxPoolSize=3 2024-11-15T22:15:29,726 DEBUG [RS:0;e611192d6313:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0, corePoolSize=3, maxPoolSize=3 2024-11-15T22:15:29,727 INFO [RS:0;e611192d6313:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:29,727 INFO [RS:0;e611192d6313:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:29,727 INFO [RS:0;e611192d6313:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:29,727 INFO [RS:0;e611192d6313:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:29,728 INFO [RS:0;e611192d6313:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:29,728 INFO [RS:0;e611192d6313:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,38779,1731708927034-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T22:15:29,745 INFO [RS:0;e611192d6313:38779 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T22:15:29,747 INFO [RS:0;e611192d6313:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,38779,1731708927034-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:29,747 INFO [RS:0;e611192d6313:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:29,747 INFO [RS:0;e611192d6313:38779 {}] regionserver.Replication(171): e611192d6313,38779,1731708927034 started 2024-11-15T22:15:29,769 INFO [RS:0;e611192d6313:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:29,769 INFO [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(1482): Serving as e611192d6313,38779,1731708927034, RpcServer on e611192d6313/172.17.0.3:38779, sessionid=0x10140912f520001 2024-11-15T22:15:29,770 DEBUG [RS:0;e611192d6313:38779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T22:15:29,770 DEBUG [RS:0;e611192d6313:38779 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e611192d6313,38779,1731708927034 2024-11-15T22:15:29,771 DEBUG [RS:0;e611192d6313:38779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e611192d6313,38779,1731708927034' 2024-11-15T22:15:29,771 DEBUG [RS:0;e611192d6313:38779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T22:15:29,772 DEBUG [RS:0;e611192d6313:38779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T22:15:29,773 DEBUG [RS:0;e611192d6313:38779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T22:15:29,773 DEBUG [RS:0;e611192d6313:38779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T22:15:29,773 DEBUG [RS:0;e611192d6313:38779 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e611192d6313,38779,1731708927034 2024-11-15T22:15:29,773 DEBUG [RS:0;e611192d6313:38779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e611192d6313,38779,1731708927034' 2024-11-15T22:15:29,773 DEBUG [RS:0;e611192d6313:38779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T22:15:29,774 DEBUG [RS:0;e611192d6313:38779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T22:15:29,774 DEBUG [RS:0;e611192d6313:38779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T22:15:29,775 INFO [RS:0;e611192d6313:38779 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T22:15:29,775 INFO [RS:0;e611192d6313:38779 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T22:15:29,887 INFO [RS:0;e611192d6313:38779 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T22:15:29,892 INFO [RS:0;e611192d6313:38779 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e611192d6313%2C38779%2C1731708927034, suffix=, logDir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/WALs/e611192d6313,38779,1731708927034, archiveDir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/oldWALs, maxLogs=32 2024-11-15T22:15:29,916 DEBUG [RS:0;e611192d6313:38779 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/WALs/e611192d6313,38779,1731708927034/e611192d6313%2C38779%2C1731708927034.1731708929896, exclude list is [], retry=0 2024-11-15T22:15:29,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38455,DS-fbaa7e2b-1311-447c-a7c5-ccdf11aa0038,DISK] 2024-11-15T22:15:29,924 INFO [RS:0;e611192d6313:38779 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/WALs/e611192d6313,38779,1731708927034/e611192d6313%2C38779%2C1731708927034.1731708929896 2024-11-15T22:15:29,925 DEBUG [RS:0;e611192d6313:38779 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40791:40791)] 2024-11-15T22:15:29,998 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T22:15:29,999 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e 2024-11-15T22:15:30,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741833_1009 (size=32) 2024-11-15T22:15:30,418 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:15:30,424 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T22:15:30,426 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T22:15:30,426 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:30,427 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:15:30,428 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T22:15:30,430 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T22:15:30,430 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:30,431 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:15:30,431 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T22:15:30,433 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T22:15:30,434 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:30,434 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:15:30,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T22:15:30,437 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T22:15:30,438 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:30,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:15:30,439 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T22:15:30,441 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740 2024-11-15T22:15:30,441 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740 2024-11-15T22:15:30,444 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T22:15:30,444 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T22:15:30,445 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:15:30,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T22:15:30,452 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:15:30,453 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70625099, jitterRate=0.052395984530448914}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:15:30,457 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731708930418Initializing all the Stores at 1731708930423 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708930423Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708930423Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708930423Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708930423Cleaning up temporary data from old regions at 1731708930444 (+21 ms)Region opened successfully at 1731708930457 (+13 ms) 2024-11-15T22:15:30,457 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T22:15:30,457 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T22:15:30,457 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T22:15:30,457 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T22:15:30,457 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T22:15:30,459 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T22:15:30,459 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731708930457Disabling compacts and flushes for region at 1731708930457Disabling writes for close at 1731708930457Writing region close event to WAL at 1731708930458 (+1 ms)Closed at 1731708930458 2024-11-15T22:15:30,462 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T22:15:30,462 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T22:15:30,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T22:15:30,475 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:15:30,478 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T22:15:30,635 DEBUG [e611192d6313:34969 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-15T22:15:30,648 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:15:30,655 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38779,1731708927034, state=OPENING 2024-11-15T22:15:30,681 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T22:15:30,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:15:30,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:15:30,694 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:15:30,694 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:15:30,697 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:15:30,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:15:30,885 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T22:15:30,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55709, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T22:15:30,900 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T22:15:30,900 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T22:15:30,901 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T22:15:30,904 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e611192d6313%2C38779%2C1731708927034.meta, suffix=.meta, logDir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/WALs/e611192d6313,38779,1731708927034, archiveDir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/oldWALs, maxLogs=32 2024-11-15T22:15:30,918 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/WALs/e611192d6313,38779,1731708927034/e611192d6313%2C38779%2C1731708927034.meta.1731708930906.meta, exclude list is [], retry=0 2024-11-15T22:15:30,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38455,DS-fbaa7e2b-1311-447c-a7c5-ccdf11aa0038,DISK] 2024-11-15T22:15:30,925 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/WALs/e611192d6313,38779,1731708927034/e611192d6313%2C38779%2C1731708927034.meta.1731708930906.meta 2024-11-15T22:15:30,925 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40791:40791)] 2024-11-15T22:15:30,926 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:15:30,927 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T22:15:30,929 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T22:15:30,933 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T22:15:30,937 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T22:15:30,937 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:15:30,938 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T22:15:30,938 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T22:15:30,941 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T22:15:30,942 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T22:15:30,942 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:30,943 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:15:30,943 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T22:15:30,945 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T22:15:30,945 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:30,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:15:30,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T22:15:30,948 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T22:15:30,949 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:30,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:15:30,950 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T22:15:30,951 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T22:15:30,951 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:30,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:15:30,953 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T22:15:30,954 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740 2024-11-15T22:15:30,957 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740 2024-11-15T22:15:30,959 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T22:15:30,960 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T22:15:30,961 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:15:30,964 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T22:15:30,966 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62245305, jitterRate=-0.07247267663478851}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:15:30,966 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T22:15:30,968 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731708930938Writing region info on filesystem at 1731708930938Initializing all the Stores at 1731708930940 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708930940Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708930940Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708930940Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708930940Cleaning up temporary data from old regions at 1731708930960 (+20 ms)Running coprocessor post-open hooks at 1731708930966 (+6 ms)Region opened successfully at 1731708930968 (+2 ms) 2024-11-15T22:15:30,975 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731708930878 2024-11-15T22:15:30,986 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T22:15:30,986 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T22:15:30,988 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:15:30,990 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38779,1731708927034, state=OPEN 2024-11-15T22:15:31,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:15:31,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:15:31,068 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:15:31,068 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:15:31,069 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:31,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T22:15:31,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38779,1731708927034 in 369 msec 2024-11-15T22:15:31,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T22:15:31,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 612 msec 2024-11-15T22:15:31,088 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T22:15:31,088 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T22:15:31,107 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:15:31,108 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:15:31,126 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:15:31,129 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51713, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:15:31,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6570 sec 2024-11-15T22:15:31,150 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731708931150, completionTime=-1 2024-11-15T22:15:31,153 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-15T22:15:31,153 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:15:31,177 INFO [master/e611192d6313:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-15T22:15:31,177 INFO [master/e611192d6313:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708991177 2024-11-15T22:15:31,177 INFO [master/e611192d6313:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709051177 2024-11-15T22:15:31,178 INFO [master/e611192d6313:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 24 msec 2024-11-15T22:15:31,179 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,34969,1731708925722-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:31,180 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,34969,1731708925722-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:31,180 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,34969,1731708925722-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:31,181 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e611192d6313:34969, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:31,181 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:31,182 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:31,186 DEBUG [master/e611192d6313:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T22:15:31,227 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.027sec 2024-11-15T22:15:31,228 INFO [master/e611192d6313:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T22:15:31,230 INFO [master/e611192d6313:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T22:15:31,231 INFO [master/e611192d6313:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T22:15:31,232 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T22:15:31,232 INFO [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T22:15:31,233 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,34969,1731708925722-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T22:15:31,234 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,34969,1731708925722-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T22:15:31,246 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T22:15:31,247 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T22:15:31,248 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,34969,1731708925722-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:15:31,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4471152c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:31,333 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T22:15:31,333 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T22:15:31,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:15:31,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:15:31,345 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:15:31,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:15:31,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:15:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c924b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:15:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:15:31,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:31,355 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:15:31,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e7f88e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:31,358 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:15:31,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:15:31,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:15:31,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:15:31,388 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e611192d6313,34969,1731708925722 2024-11-15T22:15:31,397 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=653, ProcessCount=11, AvailableMemoryMB=1301 2024-11-15T22:15:31,404 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T22:15:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e611192d6313,34969,1731708925722 2024-11-15T22:15:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54ffeb4e 2024-11-15T22:15:31,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T22:15:31,416 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T22:15:31,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:15:31,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:15:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T22:15:31,432 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T22:15:31,434 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:31,434 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-15T22:15:31,436 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T22:15:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:15:31,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741835_1011 (size=1061) 2024-11-15T22:15:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:15:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:15:31,855 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e 2024-11-15T22:15:31,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741836_1012 (size=53) 2024-11-15T22:15:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:15:32,268 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:15:32,268 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0cf44e7263c9528c718ed800ce4e85e9, disabling compactions & flushes 2024-11-15T22:15:32,268 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:32,268 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:32,268 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. after waiting 0 ms 2024-11-15T22:15:32,268 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:32,268 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:32,268 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0cf44e7263c9528c718ed800ce4e85e9: Waiting for close lock at 1731708932268Disabling compacts and flushes for region at 1731708932268Disabling writes for close at 1731708932268Writing region close event to WAL at 1731708932268Closed at 1731708932268 2024-11-15T22:15:32,271 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T22:15:32,277 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731708932271"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708932271"}]},"ts":"1731708932271"} 2024-11-15T22:15:32,282 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T22:15:32,285 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T22:15:32,287 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708932285"}]},"ts":"1731708932285"} 2024-11-15T22:15:32,292 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T22:15:32,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0cf44e7263c9528c718ed800ce4e85e9, ASSIGN}] 2024-11-15T22:15:32,296 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0cf44e7263c9528c718ed800ce4e85e9, ASSIGN 2024-11-15T22:15:32,298 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0cf44e7263c9528c718ed800ce4e85e9, ASSIGN; state=OFFLINE, location=e611192d6313,38779,1731708927034; forceNewPlan=false, retain=false 2024-11-15T22:15:32,451 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0cf44e7263c9528c718ed800ce4e85e9, regionState=OPENING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:15:32,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0cf44e7263c9528c718ed800ce4e85e9, ASSIGN because future has completed 2024-11-15T22:15:32,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:15:32,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:15:32,626 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:32,626 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:15:32,627 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:32,627 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:15:32,627 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:32,627 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:32,630 INFO [StoreOpener-0cf44e7263c9528c718ed800ce4e85e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:32,633 INFO [StoreOpener-0cf44e7263c9528c718ed800ce4e85e9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:15:32,634 INFO [StoreOpener-0cf44e7263c9528c718ed800ce4e85e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0cf44e7263c9528c718ed800ce4e85e9 columnFamilyName A 2024-11-15T22:15:32,634 DEBUG [StoreOpener-0cf44e7263c9528c718ed800ce4e85e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:32,635 INFO [StoreOpener-0cf44e7263c9528c718ed800ce4e85e9-1 {}] regionserver.HStore(327): Store=0cf44e7263c9528c718ed800ce4e85e9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:15:32,636 INFO [StoreOpener-0cf44e7263c9528c718ed800ce4e85e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:32,638 INFO [StoreOpener-0cf44e7263c9528c718ed800ce4e85e9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:15:32,638 INFO [StoreOpener-0cf44e7263c9528c718ed800ce4e85e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0cf44e7263c9528c718ed800ce4e85e9 columnFamilyName B 2024-11-15T22:15:32,638 DEBUG [StoreOpener-0cf44e7263c9528c718ed800ce4e85e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:32,639 INFO [StoreOpener-0cf44e7263c9528c718ed800ce4e85e9-1 {}] regionserver.HStore(327): Store=0cf44e7263c9528c718ed800ce4e85e9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:15:32,639 INFO [StoreOpener-0cf44e7263c9528c718ed800ce4e85e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:32,641 INFO [StoreOpener-0cf44e7263c9528c718ed800ce4e85e9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:15:32,641 INFO [StoreOpener-0cf44e7263c9528c718ed800ce4e85e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0cf44e7263c9528c718ed800ce4e85e9 columnFamilyName C 2024-11-15T22:15:32,641 DEBUG [StoreOpener-0cf44e7263c9528c718ed800ce4e85e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:32,642 INFO [StoreOpener-0cf44e7263c9528c718ed800ce4e85e9-1 {}] regionserver.HStore(327): Store=0cf44e7263c9528c718ed800ce4e85e9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:15:32,643 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:32,643 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:32,644 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:32,645 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:32,647 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:32,647 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:32,648 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:15:32,650 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:32,653 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:15:32,654 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0cf44e7263c9528c718ed800ce4e85e9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74866520, jitterRate=0.11559808254241943}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:15:32,654 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:32,655 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0cf44e7263c9528c718ed800ce4e85e9: Running coprocessor pre-open hook at 1731708932627Writing region info on filesystem at 1731708932627Initializing all the Stores at 1731708932629 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708932629Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708932629Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708932629Cleaning up temporary data from old regions at 1731708932647 (+18 ms)Running coprocessor post-open hooks at 1731708932654 (+7 ms)Region opened successfully at 1731708932655 (+1 ms) 2024-11-15T22:15:32,657 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., pid=6, masterSystemTime=1731708932614 2024-11-15T22:15:32,662 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:32,662 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:32,662 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0cf44e7263c9528c718ed800ce4e85e9, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:15:32,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:15:32,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T22:15:32,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 in 209 msec 2024-11-15T22:15:32,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T22:15:32,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0cf44e7263c9528c718ed800ce4e85e9, ASSIGN in 380 msec 2024-11-15T22:15:32,679 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T22:15:32,679 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708932679"}]},"ts":"1731708932679"} 2024-11-15T22:15:32,683 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T22:15:32,685 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T22:15:32,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2600 sec 2024-11-15T22:15:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:15:33,603 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:15:33,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75877970, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,610 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:15:33,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:15:33,612 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:15:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:15:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:15:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46cd3e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:15:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:15:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:33,617 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:15:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c82fed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,620 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:15:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:15:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:15:33,626 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:15:33,629 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T22:15:33,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e611192d6313,34969,1731708925722 2024-11-15T22:15:33,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@59e86fe5 2024-11-15T22:15:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T22:15:33,637 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T22:15:33,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a0c60c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:15:33,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:15:33,644 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:15:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:15:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:15:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:15:33,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:15:33,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:33,648 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:15:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7685e9fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:15:33,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:15:33,652 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:15:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:15:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:15:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf4e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:15:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:15:33,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:33,658 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:15:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e3eb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:15:33,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:15:33,663 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:15:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:15:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:15:33,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:15:33,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:15:33,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:33,668 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:15:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4d8404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:15:33,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:15:33,676 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:15:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:15:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:15:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0e6697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:15:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:15:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:33,682 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:15:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7ca9c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382062a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:15:33,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:15:33,687 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:15:33,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:15:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:15:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70343832, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:15:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:15:33,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:33,690 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:15:33,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5127c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53cb06a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:15:33,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:15:33,695 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:15:33,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:15:33,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:15:33,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a9c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:15:33,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:15:33,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:33,700 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:15:33,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b096a35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ef992a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:15:33,703 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:15:33,704 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:15:33,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:15:33,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:15:33,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d111af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:15:33,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:15:33,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:33,707 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:15:33,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49742bbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b28440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:15:33,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:15:33,711 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:15:33,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:15:33,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:15:33,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e406351, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:15:33,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:15:33,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:33,715 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:15:33,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44bee5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64696a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,719 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:15:33,719 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:15:33,720 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:15:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:15:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:15:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e186e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:15:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:15:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:33,724 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:15:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2171b18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:15:33,731 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:15:33,731 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:15:33,731 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:15:33,731 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:15:33,732 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:15:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:15:33,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:15:33,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:15:33,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:15:33,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:15:33,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:15:33,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:15:33,736 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:15:33,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:15:33,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:15:33,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:15:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:15:33,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:15:33,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:15:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:15:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:15:33,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:15:33,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:15:33,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:15:33,743 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:15:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:15:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:15:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-15T22:15:33,752 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:15:33,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:15:33,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:15:33,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:15:33,756 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:15:33,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:15:33,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:15:33,759 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:15:33,759 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:15:33,760 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:15:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:15:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:15:33,762 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:15:33,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:15:33,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:15:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T22:15:33,767 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:15:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:15:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:15:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:15:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:15:33,771 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:15:33,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:15:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T22:15:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:33,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T22:15:33,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:33,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:33,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:33,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:33,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:33,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:33,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:15:33,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:33,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:33,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:33,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708994019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708994020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708994024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T22:15:34,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708994047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708994048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/2b535aeef3fa44c5999f61dff4b42e4c is 50, key is test_row_0/A:col10/1731708933861/Put/seqid=0 2024-11-15T22:15:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741837_1013 (size=14341) 2024-11-15T22:15:34,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/2b535aeef3fa44c5999f61dff4b42e4c 2024-11-15T22:15:34,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:15:34,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:34,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:34,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:34,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708994213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708994215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708994216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708994223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/c72061c580bc40b4a106ebf8a3f240eb is 50, key is test_row_0/B:col10/1731708933861/Put/seqid=0 2024-11-15T22:15:34,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708994223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741838_1014 (size=12001) 2024-11-15T22:15:34,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/c72061c580bc40b4a106ebf8a3f240eb 2024-11-15T22:15:34,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:15:34,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/cda921b7380d4f50830c7f033019deb8 is 50, key is test_row_0/C:col10/1731708933861/Put/seqid=0 2024-11-15T22:15:34,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:34,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:34,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:34,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741839_1015 (size=12001) 2024-11-15T22:15:34,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/cda921b7380d4f50830c7f033019deb8 2024-11-15T22:15:34,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T22:15:34,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/2b535aeef3fa44c5999f61dff4b42e4c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2b535aeef3fa44c5999f61dff4b42e4c 2024-11-15T22:15:34,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2b535aeef3fa44c5999f61dff4b42e4c, entries=200, sequenceid=15, filesize=14.0 K 2024-11-15T22:15:34,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/c72061c580bc40b4a106ebf8a3f240eb as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c72061c580bc40b4a106ebf8a3f240eb 2024-11-15T22:15:34,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c72061c580bc40b4a106ebf8a3f240eb, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T22:15:34,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/cda921b7380d4f50830c7f033019deb8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/cda921b7380d4f50830c7f033019deb8 2024-11-15T22:15:34,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708994444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708994445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708994445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708994452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/cda921b7380d4f50830c7f033019deb8, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T22:15:34,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0cf44e7263c9528c718ed800ce4e85e9 in 586ms, sequenceid=15, compaction requested=false 2024-11-15T22:15:34,474 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T22:15:34,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:34,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T22:15:34,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:34,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:34,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:34,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:34,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:34,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:34,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/a6413c0082af4b0ca2ac852b94da1fd2 is 50, key is test_row_0/A:col10/1731708934017/Put/seqid=0 2024-11-15T22:15:34,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:15:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:34,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:34,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:34,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741840_1016 (size=14341) 2024-11-15T22:15:34,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/a6413c0082af4b0ca2ac852b94da1fd2 2024-11-15T22:15:34,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/1af2b849e8554562a706c818e522fe3e is 50, key is test_row_0/B:col10/1731708934017/Put/seqid=0 2024-11-15T22:15:34,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741841_1017 (size=12001) 2024-11-15T22:15:34,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/1af2b849e8554562a706c818e522fe3e 2024-11-15T22:15:34,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/ea26ed682ae042308c336c9f482e0c53 is 50, key is test_row_0/C:col10/1731708934017/Put/seqid=0 2024-11-15T22:15:34,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708994653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741842_1018 (size=12001) 2024-11-15T22:15:34,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/ea26ed682ae042308c336c9f482e0c53 2024-11-15T22:15:34,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:15:34,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:34,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/a6413c0082af4b0ca2ac852b94da1fd2 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/a6413c0082af4b0ca2ac852b94da1fd2 2024-11-15T22:15:34,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/a6413c0082af4b0ca2ac852b94da1fd2, entries=200, sequenceid=34, filesize=14.0 K 2024-11-15T22:15:34,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/1af2b849e8554562a706c818e522fe3e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/1af2b849e8554562a706c818e522fe3e 2024-11-15T22:15:34,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/1af2b849e8554562a706c818e522fe3e, entries=150, sequenceid=34, filesize=11.7 K 2024-11-15T22:15:34,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/ea26ed682ae042308c336c9f482e0c53 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ea26ed682ae042308c336c9f482e0c53 2024-11-15T22:15:34,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ea26ed682ae042308c336c9f482e0c53, entries=150, sequenceid=34, filesize=11.7 K 2024-11-15T22:15:34,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 0cf44e7263c9528c718ed800ce4e85e9 in 270ms, sequenceid=34, compaction requested=false 2024-11-15T22:15:34,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:34,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T22:15:34,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:34,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:34,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:34,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:34,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:34,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:34,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/7aa245bb4a3c4e989a9868d7c9f463a7 is 50, key is test_row_0/A:col10/1731708934766/Put/seqid=0 2024-11-15T22:15:34,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741843_1019 (size=14341) 2024-11-15T22:15:34,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/7aa245bb4a3c4e989a9868d7c9f463a7 2024-11-15T22:15:34,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:15:34,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:34,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:34,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:34,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708994836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708994840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708994840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708994841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708994845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/23a4272a2b9d41f8a2918da955cfa499 is 50, key is test_row_0/B:col10/1731708934766/Put/seqid=0 2024-11-15T22:15:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T22:15:34,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741844_1020 (size=12001) 2024-11-15T22:15:34,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708994962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708994963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708994963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708994972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708994973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:15:34,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:34,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:34,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:34,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:34,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:34,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:35,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:15:35,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:35,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:35,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:35,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:35,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:35,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708995172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708995182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708995182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708995195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708995202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:15:35,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:35,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:35,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:35,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:35,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:35,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/23a4272a2b9d41f8a2918da955cfa499 2024-11-15T22:15:35,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/f7102f89913a4e63b9100af3da21a47c is 50, key is test_row_0/C:col10/1731708934766/Put/seqid=0 2024-11-15T22:15:35,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741845_1021 (size=12001) 2024-11-15T22:15:35,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/f7102f89913a4e63b9100af3da21a47c 2024-11-15T22:15:35,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/7aa245bb4a3c4e989a9868d7c9f463a7 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7aa245bb4a3c4e989a9868d7c9f463a7 2024-11-15T22:15:35,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7aa245bb4a3c4e989a9868d7c9f463a7, entries=200, sequenceid=48, filesize=14.0 K 2024-11-15T22:15:35,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/23a4272a2b9d41f8a2918da955cfa499 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/23a4272a2b9d41f8a2918da955cfa499 2024-11-15T22:15:35,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/23a4272a2b9d41f8a2918da955cfa499, entries=150, sequenceid=48, filesize=11.7 K 2024-11-15T22:15:35,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/f7102f89913a4e63b9100af3da21a47c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/f7102f89913a4e63b9100af3da21a47c 2024-11-15T22:15:35,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/f7102f89913a4e63b9100af3da21a47c, entries=150, sequenceid=48, filesize=11.7 K 2024-11-15T22:15:35,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0cf44e7263c9528c718ed800ce4e85e9 in 668ms, sequenceid=48, compaction requested=true 2024-11-15T22:15:35,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:35,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:15:35,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:35,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:15:35,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:35,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:15:35,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:35,443 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:35,447 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:35,449 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/B is initiating minor compaction (all files) 2024-11-15T22:15:35,449 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:35,450 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/B in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:35,450 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c72061c580bc40b4a106ebf8a3f240eb, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/1af2b849e8554562a706c818e522fe3e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/23a4272a2b9d41f8a2918da955cfa499] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=35.2 K 2024-11-15T22:15:35,452 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c72061c580bc40b4a106ebf8a3f240eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731708933861 2024-11-15T22:15:35,452 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:35,453 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/A is initiating minor compaction (all files) 2024-11-15T22:15:35,453 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/A in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:35,453 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2b535aeef3fa44c5999f61dff4b42e4c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/a6413c0082af4b0ca2ac852b94da1fd2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7aa245bb4a3c4e989a9868d7c9f463a7] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=42.0 K 2024-11-15T22:15:35,454 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 1af2b849e8554562a706c818e522fe3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731708934008 2024-11-15T22:15:35,454 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b535aeef3fa44c5999f61dff4b42e4c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731708933861 2024-11-15T22:15:35,455 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 23a4272a2b9d41f8a2918da955cfa499, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731708934595 2024-11-15T22:15:35,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:15:35,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:35,457 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T22:15:35,457 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6413c0082af4b0ca2ac852b94da1fd2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731708934008 2024-11-15T22:15:35,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:35,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:35,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:35,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:35,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:35,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:35,458 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aa245bb4a3c4e989a9868d7c9f463a7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731708934595 2024-11-15T22:15:35,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/184691cead8244a1a473957c19f00ef1 is 50, key is test_row_0/A:col10/1731708934827/Put/seqid=0 2024-11-15T22:15:35,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:35,517 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#A#compaction#11 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:35,517 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#B#compaction#10 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:35,518 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/950455225d2340588ae5dfa972d095e2 is 50, key is test_row_0/B:col10/1731708934766/Put/seqid=0 2024-11-15T22:15:35,518 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/2872caa7bfa24880979cf9aef9689257 is 50, key is test_row_0/A:col10/1731708934766/Put/seqid=0 2024-11-15T22:15:35,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741846_1022 (size=12001) 2024-11-15T22:15:35,529 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/184691cead8244a1a473957c19f00ef1 2024-11-15T22:15:35,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741848_1024 (size=12104) 2024-11-15T22:15:35,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741847_1023 (size=12104) 2024-11-15T22:15:35,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/4d882ef8a9cd4881808bdfe85220ddc3 is 50, key is test_row_0/B:col10/1731708934827/Put/seqid=0 2024-11-15T22:15:35,575 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/950455225d2340588ae5dfa972d095e2 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/950455225d2340588ae5dfa972d095e2 2024-11-15T22:15:35,577 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/2872caa7bfa24880979cf9aef9689257 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2872caa7bfa24880979cf9aef9689257 2024-11-15T22:15:35,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741849_1025 (size=12001) 2024-11-15T22:15:35,602 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/A of 0cf44e7263c9528c718ed800ce4e85e9 into 2872caa7bfa24880979cf9aef9689257(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:35,603 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:35,606 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/A, priority=13, startTime=1731708935439; duration=0sec 2024-11-15T22:15:35,606 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:35,606 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:A 2024-11-15T22:15:35,607 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:35,607 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/B of 0cf44e7263c9528c718ed800ce4e85e9 into 950455225d2340588ae5dfa972d095e2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:35,607 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:35,607 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/B, priority=13, startTime=1731708935442; duration=0sec 2024-11-15T22:15:35,608 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:35,608 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:B 2024-11-15T22:15:35,610 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:35,611 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/C is initiating minor compaction (all files) 2024-11-15T22:15:35,611 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/C in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:35,611 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/cda921b7380d4f50830c7f033019deb8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ea26ed682ae042308c336c9f482e0c53, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/f7102f89913a4e63b9100af3da21a47c] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=35.2 K 2024-11-15T22:15:35,612 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting cda921b7380d4f50830c7f033019deb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731708933861 2024-11-15T22:15:35,612 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea26ed682ae042308c336c9f482e0c53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731708934008 2024-11-15T22:15:35,616 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7102f89913a4e63b9100af3da21a47c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731708934595 2024-11-15T22:15:35,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708995603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708995605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708995608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708995609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708995619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,646 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#C#compaction#13 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:35,647 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/ed5dfcaab8f8440981b49e9e70c0f50a is 50, key is test_row_0/C:col10/1731708934766/Put/seqid=0 2024-11-15T22:15:35,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741850_1026 (size=12104) 2024-11-15T22:15:35,686 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/ed5dfcaab8f8440981b49e9e70c0f50a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ed5dfcaab8f8440981b49e9e70c0f50a 2024-11-15T22:15:35,691 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T22:15:35,705 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/C of 0cf44e7263c9528c718ed800ce4e85e9 into ed5dfcaab8f8440981b49e9e70c0f50a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:35,705 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:35,705 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/C, priority=13, startTime=1731708935442; duration=0sec 2024-11-15T22:15:35,705 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:35,705 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:C 2024-11-15T22:15:35,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708995722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708995733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708995734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708995734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708995734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T22:15:35,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708995943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708995943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708995953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708995953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708995953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:35,996 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/4d882ef8a9cd4881808bdfe85220ddc3 2024-11-15T22:15:36,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/ec72505521fe4eef8415ab989981f95c is 50, key is test_row_0/C:col10/1731708934827/Put/seqid=0 2024-11-15T22:15:36,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741851_1027 (size=12001) 2024-11-15T22:15:36,057 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/ec72505521fe4eef8415ab989981f95c 2024-11-15T22:15:36,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/184691cead8244a1a473957c19f00ef1 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/184691cead8244a1a473957c19f00ef1 2024-11-15T22:15:36,087 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/184691cead8244a1a473957c19f00ef1, entries=150, sequenceid=66, filesize=11.7 K 2024-11-15T22:15:36,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/4d882ef8a9cd4881808bdfe85220ddc3 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/4d882ef8a9cd4881808bdfe85220ddc3 2024-11-15T22:15:36,108 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/4d882ef8a9cd4881808bdfe85220ddc3, entries=150, sequenceid=66, filesize=11.7 K 2024-11-15T22:15:36,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/ec72505521fe4eef8415ab989981f95c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ec72505521fe4eef8415ab989981f95c 2024-11-15T22:15:36,125 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ec72505521fe4eef8415ab989981f95c, entries=150, sequenceid=66, filesize=11.7 K 2024-11-15T22:15:36,127 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0cf44e7263c9528c718ed800ce4e85e9 in 671ms, sequenceid=66, compaction requested=false 2024-11-15T22:15:36,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:36,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:36,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-15T22:15:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-15T22:15:36,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T22:15:36,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3700 sec 2024-11-15T22:15:36,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3980 sec 2024-11-15T22:15:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:36,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T22:15:36,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:36,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:36,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:36,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:36,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:36,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:36,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/05494c0eeb1043e29858dee9e5eba4fb is 50, key is test_row_0/A:col10/1731708935614/Put/seqid=0 2024-11-15T22:15:36,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741852_1028 (size=12001) 2024-11-15T22:15:36,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/05494c0eeb1043e29858dee9e5eba4fb 2024-11-15T22:15:36,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/d4a4740bc1bf4b83b654a893886bc6fe is 50, key is test_row_0/B:col10/1731708935614/Put/seqid=0 2024-11-15T22:15:36,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708996312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741853_1029 (size=12001) 2024-11-15T22:15:36,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708996318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708996319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/d4a4740bc1bf4b83b654a893886bc6fe 2024-11-15T22:15:36,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708996322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708996322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/11c2f79abed04086b25d4d1d89ea8cc1 is 50, key is test_row_0/C:col10/1731708935614/Put/seqid=0 2024-11-15T22:15:36,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741854_1030 (size=12001) 2024-11-15T22:15:36,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/11c2f79abed04086b25d4d1d89ea8cc1 2024-11-15T22:15:36,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/05494c0eeb1043e29858dee9e5eba4fb as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/05494c0eeb1043e29858dee9e5eba4fb 2024-11-15T22:15:36,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/05494c0eeb1043e29858dee9e5eba4fb, entries=150, sequenceid=85, filesize=11.7 K 2024-11-15T22:15:36,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/d4a4740bc1bf4b83b654a893886bc6fe as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d4a4740bc1bf4b83b654a893886bc6fe 2024-11-15T22:15:36,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d4a4740bc1bf4b83b654a893886bc6fe, entries=150, sequenceid=85, filesize=11.7 K 2024-11-15T22:15:36,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/11c2f79abed04086b25d4d1d89ea8cc1 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/11c2f79abed04086b25d4d1d89ea8cc1 2024-11-15T22:15:36,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/11c2f79abed04086b25d4d1d89ea8cc1, entries=150, sequenceid=85, filesize=11.7 K 2024-11-15T22:15:36,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 0cf44e7263c9528c718ed800ce4e85e9 in 167ms, sequenceid=85, compaction requested=true 2024-11-15T22:15:36,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:36,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:15:36,435 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:36,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:36,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:15:36,435 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:36,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:36,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:15:36,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:36,438 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:36,438 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/B is initiating minor compaction (all files) 2024-11-15T22:15:36,438 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:36,438 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/B in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:36,438 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/A is initiating minor compaction (all files) 2024-11-15T22:15:36,438 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/A in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:36,438 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/950455225d2340588ae5dfa972d095e2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/4d882ef8a9cd4881808bdfe85220ddc3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d4a4740bc1bf4b83b654a893886bc6fe] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=35.3 K 2024-11-15T22:15:36,439 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2872caa7bfa24880979cf9aef9689257, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/184691cead8244a1a473957c19f00ef1, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/05494c0eeb1043e29858dee9e5eba4fb] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=35.3 K 2024-11-15T22:15:36,440 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 950455225d2340588ae5dfa972d095e2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731708934595 2024-11-15T22:15:36,440 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2872caa7bfa24880979cf9aef9689257, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731708934595 2024-11-15T22:15:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:36,445 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d882ef8a9cd4881808bdfe85220ddc3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731708934814 2024-11-15T22:15:36,445 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 184691cead8244a1a473957c19f00ef1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731708934814 2024-11-15T22:15:36,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T22:15:36,446 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d4a4740bc1bf4b83b654a893886bc6fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731708935614 2024-11-15T22:15:36,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:36,447 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05494c0eeb1043e29858dee9e5eba4fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731708935614 2024-11-15T22:15:36,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:36,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:36,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:36,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:36,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:36,449 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T22:15:36,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/afb416507734432abbc50bac19426742 is 50, key is test_row_0/A:col10/1731708936438/Put/seqid=0 2024-11-15T22:15:36,488 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:36,490 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/c025da8e7c674fac94728da69b7b106e is 50, key is test_row_0/B:col10/1731708935614/Put/seqid=0 2024-11-15T22:15:36,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708996480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708996481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708996476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708996494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,503 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#A#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:36,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,503 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/0e09eab9ffe34e1584a728936830f6b8 is 50, key is test_row_0/A:col10/1731708935614/Put/seqid=0 2024-11-15T22:15:36,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708996497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741856_1032 (size=12207) 2024-11-15T22:15:36,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741855_1031 (size=14341) 2024-11-15T22:15:36,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/afb416507734432abbc50bac19426742 2024-11-15T22:15:36,546 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/c025da8e7c674fac94728da69b7b106e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c025da8e7c674fac94728da69b7b106e 2024-11-15T22:15:36,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741857_1033 (size=12207) 2024-11-15T22:15:36,561 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T22:15:36,562 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T22:15:36,565 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/B of 0cf44e7263c9528c718ed800ce4e85e9 into c025da8e7c674fac94728da69b7b106e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:36,565 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:36,565 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/B, priority=13, startTime=1731708936435; duration=0sec 2024-11-15T22:15:36,566 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:36,566 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:B 2024-11-15T22:15:36,566 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:36,568 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T22:15:36,568 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:36,568 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T22:15:36,568 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/C is initiating minor compaction (all files) 2024-11-15T22:15:36,569 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/C in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:36,569 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T22:15:36,569 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ed5dfcaab8f8440981b49e9e70c0f50a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ec72505521fe4eef8415ab989981f95c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/11c2f79abed04086b25d4d1d89ea8cc1] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=35.3 K 2024-11-15T22:15:36,569 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T22:15:36,570 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting ed5dfcaab8f8440981b49e9e70c0f50a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731708934595 2024-11-15T22:15:36,570 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T22:15:36,570 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T22:15:36,571 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting ec72505521fe4eef8415ab989981f95c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731708934814 2024-11-15T22:15:36,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/f7eb96a796134aa0bb18d278ddd7b1f9 is 50, key is test_row_0/B:col10/1731708936438/Put/seqid=0 2024-11-15T22:15:36,572 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 11c2f79abed04086b25d4d1d89ea8cc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731708935614 2024-11-15T22:15:36,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741858_1034 (size=12001) 2024-11-15T22:15:36,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708996602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708996602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,606 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#C#compaction#22 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:36,606 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/9d74eb80b2c447a08f0fc1dee997eca6 is 50, key is test_row_0/C:col10/1731708935614/Put/seqid=0 2024-11-15T22:15:36,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708996612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708996613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708996613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741859_1035 (size=12207) 2024-11-15T22:15:36,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708996811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708996811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708996822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708996832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708996833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:36,980 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/0e09eab9ffe34e1584a728936830f6b8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/0e09eab9ffe34e1584a728936830f6b8 2024-11-15T22:15:37,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/f7eb96a796134aa0bb18d278ddd7b1f9 2024-11-15T22:15:37,017 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/A of 0cf44e7263c9528c718ed800ce4e85e9 into 0e09eab9ffe34e1584a728936830f6b8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:37,020 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:37,020 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/A, priority=13, startTime=1731708936435; duration=0sec 2024-11-15T22:15:37,020 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:37,020 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:A 2024-11-15T22:15:37,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/8081e8abe30845cbaff05faf487da967 is 50, key is test_row_0/C:col10/1731708936438/Put/seqid=0 2024-11-15T22:15:37,058 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/9d74eb80b2c447a08f0fc1dee997eca6 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/9d74eb80b2c447a08f0fc1dee997eca6 2024-11-15T22:15:37,079 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/C of 0cf44e7263c9528c718ed800ce4e85e9 into 9d74eb80b2c447a08f0fc1dee997eca6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:37,080 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:37,080 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/C, priority=13, startTime=1731708936435; duration=0sec 2024-11-15T22:15:37,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741860_1036 (size=12001) 2024-11-15T22:15:37,080 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:37,080 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:C 2024-11-15T22:15:37,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708997132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:37,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:37,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708997133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:37,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:37,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708997142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708997142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:37,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708997151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:37,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/8081e8abe30845cbaff05faf487da967 2024-11-15T22:15:37,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/afb416507734432abbc50bac19426742 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/afb416507734432abbc50bac19426742 2024-11-15T22:15:37,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/afb416507734432abbc50bac19426742, entries=200, sequenceid=103, filesize=14.0 K 2024-11-15T22:15:37,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/f7eb96a796134aa0bb18d278ddd7b1f9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/f7eb96a796134aa0bb18d278ddd7b1f9 2024-11-15T22:15:37,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/f7eb96a796134aa0bb18d278ddd7b1f9, entries=150, sequenceid=103, filesize=11.7 K 2024-11-15T22:15:37,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/8081e8abe30845cbaff05faf487da967 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8081e8abe30845cbaff05faf487da967 2024-11-15T22:15:37,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8081e8abe30845cbaff05faf487da967, entries=150, sequenceid=103, filesize=11.7 K 2024-11-15T22:15:37,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0cf44e7263c9528c718ed800ce4e85e9 in 1121ms, sequenceid=103, compaction requested=false 2024-11-15T22:15:37,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:37,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:37,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T22:15:37,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:37,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:37,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:37,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:37,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:37,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:37,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/fc2c5f6f7b074ab2b8a12a63ee517714 is 50, key is test_row_0/A:col10/1731708936469/Put/seqid=0 2024-11-15T22:15:37,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741861_1037 (size=12001) 2024-11-15T22:15:37,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/fc2c5f6f7b074ab2b8a12a63ee517714 2024-11-15T22:15:37,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/c0833280e3a141d8b0ce9118c8e97887 is 50, key is test_row_0/B:col10/1731708936469/Put/seqid=0 2024-11-15T22:15:37,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708997717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:37,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708997732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:37,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:37,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708997739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:37,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708997741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:37,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:37,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708997742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:37,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:37,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741862_1038 (size=12001) 2024-11-15T22:15:37,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708997842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:37,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708997852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:37,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708997853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:37,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708997853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:37,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708997861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:37,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:37,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T22:15:37,942 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:15:37,943 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:15:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-15T22:15:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T22:15:37,946 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:15:37,948 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:15:37,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:15:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T22:15:38,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708998051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708998061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708998061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708998072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708998072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T22:15:38,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:38,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:38,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:38,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/c0833280e3a141d8b0ce9118c8e97887 2024-11-15T22:15:38,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/0349182a140442be84a67c02046c6277 is 50, key is test_row_0/C:col10/1731708936469/Put/seqid=0 2024-11-15T22:15:38,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741863_1039 (size=12001) 2024-11-15T22:15:38,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/0349182a140442be84a67c02046c6277 2024-11-15T22:15:38,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T22:15:38,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:38,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:38,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:38,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T22:15:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/fc2c5f6f7b074ab2b8a12a63ee517714 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/fc2c5f6f7b074ab2b8a12a63ee517714 2024-11-15T22:15:38,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/fc2c5f6f7b074ab2b8a12a63ee517714, entries=150, sequenceid=121, filesize=11.7 K 2024-11-15T22:15:38,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/c0833280e3a141d8b0ce9118c8e97887 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c0833280e3a141d8b0ce9118c8e97887 2024-11-15T22:15:38,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c0833280e3a141d8b0ce9118c8e97887, entries=150, sequenceid=121, filesize=11.7 K 2024-11-15T22:15:38,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/0349182a140442be84a67c02046c6277 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/0349182a140442be84a67c02046c6277 2024-11-15T22:15:38,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/0349182a140442be84a67c02046c6277, entries=150, sequenceid=121, filesize=11.7 K 2024-11-15T22:15:38,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=107.34 KB/109920 for 0cf44e7263c9528c718ed800ce4e85e9 in 727ms, sequenceid=121, compaction requested=true 2024-11-15T22:15:38,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:38,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:15:38,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:38,376 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:38,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:15:38,376 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:38,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:38,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:15:38,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:38,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T22:15:38,379 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:38,379 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/B is initiating minor compaction (all files) 2024-11-15T22:15:38,379 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/B in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:38,379 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:38,379 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c025da8e7c674fac94728da69b7b106e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/f7eb96a796134aa0bb18d278ddd7b1f9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c0833280e3a141d8b0ce9118c8e97887] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=35.4 K 2024-11-15T22:15:38,379 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/A is initiating minor compaction (all files) 2024-11-15T22:15:38,380 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/A in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:38,380 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/0e09eab9ffe34e1584a728936830f6b8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/afb416507734432abbc50bac19426742, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/fc2c5f6f7b074ab2b8a12a63ee517714] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=37.6 K 2024-11-15T22:15:38,381 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e09eab9ffe34e1584a728936830f6b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731708935614 2024-11-15T22:15:38,381 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c025da8e7c674fac94728da69b7b106e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731708935614 2024-11-15T22:15:38,382 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting afb416507734432abbc50bac19426742, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731708936270 2024-11-15T22:15:38,382 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc2c5f6f7b074ab2b8a12a63ee517714, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731708936469 2024-11-15T22:15:38,382 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting f7eb96a796134aa0bb18d278ddd7b1f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731708936270 2024-11-15T22:15:38,383 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c0833280e3a141d8b0ce9118c8e97887, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731708936469 2024-11-15T22:15:38,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708937730, max timestamp=1731708938379Num uniques 859; 2024-11-15T22:15:38,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708937730, max timestamp=1731708938379Num uniques 859; 2024-11-15T22:15:38,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:15:38,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:38,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:15:38,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731708937730, max timestamp=1731708938377Num uniques 859; 2024-11-15T22:15:38,403 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:38,401 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:38,404 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/69369e3437f94691acc18813ff5a68df is 50, key is test_row_0/A:col10/1731708936469/Put/seqid=0 2024-11-15T22:15:38,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T22:15:38,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:15:38,404 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/b78c3b250c5c40019bd90f5f1107f754 is 50, key is test_row_0/B:col10/1731708936469/Put/seqid=0 2024-11-15T22:15:38,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:38,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:15:38,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:38,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:15:38,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T22:15:38,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:38,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:38,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:38,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/fd5f84a3733543ada402d2ad6237b4e8 is 50, key is test_row_0/A:col10/1731708938379/Put/seqid=0 2024-11-15T22:15:38,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741864_1040 (size=12309) 2024-11-15T22:15:38,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741866_1042 (size=12151) 2024-11-15T22:15:38,475 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/69369e3437f94691acc18813ff5a68df as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/69369e3437f94691acc18813ff5a68df 2024-11-15T22:15:38,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741865_1041 (size=12309) 2024-11-15T22:15:38,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/fd5f84a3733543ada402d2ad6237b4e8 2024-11-15T22:15:38,496 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/b78c3b250c5c40019bd90f5f1107f754 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/b78c3b250c5c40019bd90f5f1107f754 2024-11-15T22:15:38,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708938449, max timestamp=1731708938489Num uniques 859; 2024-11-15T22:15:38,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:15:38,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/af36ff097af84538abf2448f4ce1aba6 is 50, key is test_row_0/B:col10/1731708938379/Put/seqid=0 2024-11-15T22:15:38,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708938449, max timestamp=1731708938484Num uniques 859; 2024-11-15T22:15:38,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:15:38,522 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/A of 0cf44e7263c9528c718ed800ce4e85e9 into 69369e3437f94691acc18813ff5a68df(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:38,523 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:38,522 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/B of 0cf44e7263c9528c718ed800ce4e85e9 into b78c3b250c5c40019bd90f5f1107f754(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:38,523 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:38,523 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/B, priority=13, startTime=1731708938376; duration=0sec 2024-11-15T22:15:38,523 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/A, priority=13, startTime=1731708938376; duration=0sec 2024-11-15T22:15:38,523 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:38,523 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:B 2024-11-15T22:15:38,523 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:38,523 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:A 2024-11-15T22:15:38,523 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:38,526 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:38,526 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/C is initiating minor compaction (all files) 2024-11-15T22:15:38,526 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/C in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:38,527 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/9d74eb80b2c447a08f0fc1dee997eca6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8081e8abe30845cbaff05faf487da967, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/0349182a140442be84a67c02046c6277] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=35.4 K 2024-11-15T22:15:38,528 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d74eb80b2c447a08f0fc1dee997eca6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731708935614 2024-11-15T22:15:38,529 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 8081e8abe30845cbaff05faf487da967, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731708936270 2024-11-15T22:15:38,529 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 0349182a140442be84a67c02046c6277, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731708936469 2024-11-15T22:15:38,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741867_1043 (size=12151) 2024-11-15T22:15:38,559 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#C#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:38,559 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/682f11a96b424676a70068b6c7358e20 is 50, key is test_row_0/C:col10/1731708936469/Put/seqid=0 2024-11-15T22:15:38,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708998547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708998549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T22:15:38,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T22:15:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708998565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:38,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:38,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:38,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708998567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708998567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741868_1044 (size=12309) 2024-11-15T22:15:38,598 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/682f11a96b424676a70068b6c7358e20 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/682f11a96b424676a70068b6c7358e20 2024-11-15T22:15:38,613 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/C of 0cf44e7263c9528c718ed800ce4e85e9 into 682f11a96b424676a70068b6c7358e20(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:38,614 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:38,614 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/C, priority=13, startTime=1731708938376; duration=0sec 2024-11-15T22:15:38,614 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:38,614 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:C 2024-11-15T22:15:38,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708998671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708998671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708998684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708998684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708998684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T22:15:38,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:38,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:38,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:38,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T22:15:38,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:38,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708998892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:38,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708998900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:38,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708998901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:38,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708998902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708998901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:38,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:38,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/af36ff097af84538abf2448f4ce1aba6 2024-11-15T22:15:38,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/408698720aa94510a52d6f489748c1da is 50, key is test_row_0/C:col10/1731708938379/Put/seqid=0 2024-11-15T22:15:38,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741869_1045 (size=12151) 2024-11-15T22:15:39,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T22:15:39,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:39,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:39,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:39,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:39,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T22:15:39,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T22:15:39,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:39,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:39,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:39,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:39,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:39,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708999212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:39,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708999212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:39,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708999212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:39,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:39,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708999213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708999213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:39,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:39,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T22:15:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:39,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:39,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.79 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/408698720aa94510a52d6f489748c1da 2024-11-15T22:15:39,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/fd5f84a3733543ada402d2ad6237b4e8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/fd5f84a3733543ada402d2ad6237b4e8 2024-11-15T22:15:39,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/fd5f84a3733543ada402d2ad6237b4e8, entries=150, sequenceid=143, filesize=11.9 K 2024-11-15T22:15:39,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/af36ff097af84538abf2448f4ce1aba6 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/af36ff097af84538abf2448f4ce1aba6 2024-11-15T22:15:39,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/af36ff097af84538abf2448f4ce1aba6, entries=150, sequenceid=143, filesize=11.9 K 2024-11-15T22:15:39,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/408698720aa94510a52d6f489748c1da as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/408698720aa94510a52d6f489748c1da 2024-11-15T22:15:39,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/408698720aa94510a52d6f489748c1da, entries=150, sequenceid=143, filesize=11.9 K 2024-11-15T22:15:39,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.15 KB/33945, heapSize ~85.20 KB/87240, currentSize=158.78 KB/162590 for 0cf44e7263c9528c718ed800ce4e85e9 in 1108ms, sequenceid=143, compaction requested=false 2024-11-15T22:15:39,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:39,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T22:15:39,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:39,539 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=158.78 KB heapSize=463 KB 2024-11-15T22:15:39,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:39,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:15:39,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:39,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:15:39,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:39,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:15:39,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/454e4bf242124075af6932d36a25d7fa is 50, key is test_row_0/A:col10/1731708938545/Put/seqid=0 2024-11-15T22:15:39,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741870_1046 (size=12151) 2024-11-15T22:15:39,589 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=73.80 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/454e4bf242124075af6932d36a25d7fa 2024-11-15T22:15:39,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/2bd96a1528ec474181fa192ff09035ab is 50, key is test_row_0/B:col10/1731708938545/Put/seqid=0 2024-11-15T22:15:39,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741871_1047 (size=12151) 2024-11-15T22:15:39,678 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/2bd96a1528ec474181fa192ff09035ab 2024-11-15T22:15:39,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/7a7b81ae30a542c0ac3118b8f3fa6003 is 50, key is test_row_0/C:col10/1731708938545/Put/seqid=0 2024-11-15T22:15:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:39,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:39,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741872_1048 (size=12151) 2024-11-15T22:15:39,757 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/7a7b81ae30a542c0ac3118b8f3fa6003 2024-11-15T22:15:39,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731708999741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:39,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:39,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731708999744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:39,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731708999757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:39,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731708999757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:39,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:39,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:39,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731708999764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:39,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:39,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:39,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/454e4bf242124075af6932d36a25d7fa as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/454e4bf242124075af6932d36a25d7fa 2024-11-15T22:15:39,817 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/454e4bf242124075af6932d36a25d7fa, entries=150, sequenceid=182, filesize=11.9 K 2024-11-15T22:15:39,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/2bd96a1528ec474181fa192ff09035ab as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/2bd96a1528ec474181fa192ff09035ab 2024-11-15T22:15:39,832 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/2bd96a1528ec474181fa192ff09035ab, entries=150, sequenceid=182, filesize=11.9 K 2024-11-15T22:15:39,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/7a7b81ae30a542c0ac3118b8f3fa6003 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7a7b81ae30a542c0ac3118b8f3fa6003 2024-11-15T22:15:39,845 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7a7b81ae30a542c0ac3118b8f3fa6003, entries=150, sequenceid=182, filesize=11.9 K 2024-11-15T22:15:39,848 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~158.78 KB/162590, heapSize ~462.95 KB/474064, currentSize=20.13 KB/20610 for 0cf44e7263c9528c718ed800ce4e85e9 in 309ms, sequenceid=182, compaction requested=true 2024-11-15T22:15:39,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:39,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:39,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-15T22:15:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-15T22:15:39,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-15T22:15:39,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9060 sec 2024-11-15T22:15:39,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.9290 sec 2024-11-15T22:15:39,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T22:15:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:39,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:39,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:39,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:39,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:39,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:39,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:39,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/7f5e9e461df84566b6a891aaea8d6f03 is 50, key is test_row_0/A:col10/1731708939898/Put/seqid=0 2024-11-15T22:15:39,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741873_1049 (size=12151) 2024-11-15T22:15:39,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/7f5e9e461df84566b6a891aaea8d6f03 2024-11-15T22:15:39,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/c337a1410c4e4df6b32c785d4cd3d823 is 50, key is test_row_0/B:col10/1731708939898/Put/seqid=0 2024-11-15T22:15:40,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741874_1050 (size=12151) 2024-11-15T22:15:40,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708939971, max timestamp=1731708940001Num uniques 859; 2024-11-15T22:15:40,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:15:40,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708939971, max timestamp=1731708940001Num uniques 859; 2024-11-15T22:15:40,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:15:40,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731708939971, max timestamp=1731708940001Num uniques 859; 2024-11-15T22:15:40,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T22:15:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T22:15:40,097 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:15:40,098 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:15:40,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709000086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-15T22:15:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T22:15:40,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:40,115 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:15:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709000096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:40,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709000112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:40,121 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:15:40,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:15:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T22:15:40,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709000223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:40,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709000226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:40,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709000231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:40,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T22:15:40,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:40,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:40,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:40,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:40,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:40,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/c337a1410c4e4df6b32c785d4cd3d823 2024-11-15T22:15:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T22:15:40,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T22:15:40,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:40,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:40,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:40,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:40,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:40,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:40,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709000432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:40,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:40,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709000432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:40,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709000452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:40,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:40,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/7ac80ecedad04769aac3a95ebe2684cc is 50, key is test_row_0/C:col10/1731708939898/Put/seqid=0 2024-11-15T22:15:40,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741875_1051 (size=12151) 2024-11-15T22:15:40,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T22:15:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:40,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:40,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T22:15:40,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709000751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:40,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:40,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709000752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:40,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T22:15:40,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:40,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:40,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:40,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:40,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:40,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731709000771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:40,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709000772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:40,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731709000791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:40,802 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in 0cf44e7263c9528c718ed800ce4e85e9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:15:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:40,804 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in 0cf44e7263c9528c718ed800ce4e85e9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:15:40,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:40,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:40,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/7ac80ecedad04769aac3a95ebe2684cc 2024-11-15T22:15:40,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T22:15:40,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/7f5e9e461df84566b6a891aaea8d6f03 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7f5e9e461df84566b6a891aaea8d6f03 2024-11-15T22:15:40,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:40,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:40,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:40,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:40,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:40,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7f5e9e461df84566b6a891aaea8d6f03, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T22:15:40,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/c337a1410c4e4df6b32c785d4cd3d823 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c337a1410c4e4df6b32c785d4cd3d823 2024-11-15T22:15:40,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c337a1410c4e4df6b32c785d4cd3d823, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T22:15:40,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/7ac80ecedad04769aac3a95ebe2684cc as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7ac80ecedad04769aac3a95ebe2684cc 2024-11-15T22:15:41,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7ac80ecedad04769aac3a95ebe2684cc, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T22:15:41,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for 0cf44e7263c9528c718ed800ce4e85e9 in 1120ms, sequenceid=193, compaction requested=true 2024-11-15T22:15:41,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:41,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:15:41,021 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:15:41,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:41,022 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:15:41,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:15:41,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:41,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:15:41,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:41,024 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:15:41,024 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/A is initiating minor compaction (all files) 2024-11-15T22:15:41,024 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/A in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:41,024 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/69369e3437f94691acc18813ff5a68df, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/fd5f84a3733543ada402d2ad6237b4e8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/454e4bf242124075af6932d36a25d7fa, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7f5e9e461df84566b6a891aaea8d6f03] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=47.6 K 2024-11-15T22:15:41,025 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:15:41,025 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/B is initiating minor compaction (all files) 2024-11-15T22:15:41,025 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/B in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:41,025 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/b78c3b250c5c40019bd90f5f1107f754, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/af36ff097af84538abf2448f4ce1aba6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/2bd96a1528ec474181fa192ff09035ab, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c337a1410c4e4df6b32c785d4cd3d823] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=47.6 K 2024-11-15T22:15:41,026 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting b78c3b250c5c40019bd90f5f1107f754, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731708936469 2024-11-15T22:15:41,027 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting af36ff097af84538abf2448f4ce1aba6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731708937730 2024-11-15T22:15:41,027 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69369e3437f94691acc18813ff5a68df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731708936469 2024-11-15T22:15:41,028 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bd96a1528ec474181fa192ff09035ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731708938545 2024-11-15T22:15:41,028 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd5f84a3733543ada402d2ad6237b4e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731708937730 2024-11-15T22:15:41,028 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c337a1410c4e4df6b32c785d4cd3d823, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731708939888 2024-11-15T22:15:41,028 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 454e4bf242124075af6932d36a25d7fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731708938545 2024-11-15T22:15:41,030 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f5e9e461df84566b6a891aaea8d6f03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731708939888 2024-11-15T22:15:41,101 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#A#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:41,105 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/24e51f5feac6412ea6c16b37cca0ae0d is 50, key is test_row_0/A:col10/1731708939898/Put/seqid=0 2024-11-15T22:15:41,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T22:15:41,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:41,109 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=127.08 KB heapSize=364.15 KB 2024-11-15T22:15:41,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:41,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:15:41,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:41,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:15:41,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:41,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:15:41,121 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:41,122 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/204a062f4dc3469cb3a32f6d5cccd8c5 is 50, key is test_row_0/B:col10/1731708939898/Put/seqid=0 2024-11-15T22:15:41,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/12a6c4989ff941f5a19c1ea857e4a087 is 50, key is test_row_0/A:col10/1731708940106/Put/seqid=0 2024-11-15T22:15:41,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741877_1053 (size=12151) 2024-11-15T22:15:41,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741876_1052 (size=12595) 2024-11-15T22:15:41,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741878_1054 (size=12595) 2024-11-15T22:15:41,234 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/12a6c4989ff941f5a19c1ea857e4a087 2024-11-15T22:15:41,246 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/24e51f5feac6412ea6c16b37cca0ae0d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/24e51f5feac6412ea6c16b37cca0ae0d 2024-11-15T22:15:41,258 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/A of 0cf44e7263c9528c718ed800ce4e85e9 into 24e51f5feac6412ea6c16b37cca0ae0d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:41,258 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:41,258 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/A, priority=12, startTime=1731708941021; duration=0sec 2024-11-15T22:15:41,258 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:41,258 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:A 2024-11-15T22:15:41,258 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:15:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T22:15:41,264 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:15:41,265 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/C is initiating minor compaction (all files) 2024-11-15T22:15:41,265 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/C in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:41,265 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/682f11a96b424676a70068b6c7358e20, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/408698720aa94510a52d6f489748c1da, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7a7b81ae30a542c0ac3118b8f3fa6003, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7ac80ecedad04769aac3a95ebe2684cc] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=47.6 K 2024-11-15T22:15:41,266 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 682f11a96b424676a70068b6c7358e20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731708936469 2024-11-15T22:15:41,266 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 408698720aa94510a52d6f489748c1da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731708937730 2024-11-15T22:15:41,268 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a7b81ae30a542c0ac3118b8f3fa6003, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731708938545 2024-11-15T22:15:41,269 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ac80ecedad04769aac3a95ebe2684cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731708939888 2024-11-15T22:15:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:41,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:41,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/d5b8c99767ba44df8c405f02951f0d2e is 50, key is test_row_0/B:col10/1731708940106/Put/seqid=0 2024-11-15T22:15:41,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741879_1055 (size=12151) 2024-11-15T22:15:41,295 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/d5b8c99767ba44df8c405f02951f0d2e 2024-11-15T22:15:41,315 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#C#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:41,315 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/426ad5360ce44327bd5e132373817bae is 50, key is test_row_0/C:col10/1731708939898/Put/seqid=0 2024-11-15T22:15:41,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/908d88dba55a4e36828e081c1126961f is 50, key is test_row_0/C:col10/1731708940106/Put/seqid=0 2024-11-15T22:15:41,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741880_1056 (size=12595) 2024-11-15T22:15:41,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709001335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:41,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:41,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709001335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:41,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:41,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709001341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:41,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741881_1057 (size=12151) 2024-11-15T22:15:41,368 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/908d88dba55a4e36828e081c1126961f 2024-11-15T22:15:41,383 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/426ad5360ce44327bd5e132373817bae as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/426ad5360ce44327bd5e132373817bae 2024-11-15T22:15:41,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/12a6c4989ff941f5a19c1ea857e4a087 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/12a6c4989ff941f5a19c1ea857e4a087 2024-11-15T22:15:41,410 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/C of 0cf44e7263c9528c718ed800ce4e85e9 into 426ad5360ce44327bd5e132373817bae(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:41,410 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:41,410 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/C, priority=12, startTime=1731708941022; duration=0sec 2024-11-15T22:15:41,410 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:41,411 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:C 2024-11-15T22:15:41,412 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/12a6c4989ff941f5a19c1ea857e4a087, entries=150, sequenceid=229, filesize=11.9 K 2024-11-15T22:15:41,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/d5b8c99767ba44df8c405f02951f0d2e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d5b8c99767ba44df8c405f02951f0d2e 2024-11-15T22:15:41,424 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d5b8c99767ba44df8c405f02951f0d2e, entries=150, sequenceid=229, filesize=11.9 K 2024-11-15T22:15:41,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/908d88dba55a4e36828e081c1126961f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/908d88dba55a4e36828e081c1126961f 2024-11-15T22:15:41,443 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/908d88dba55a4e36828e081c1126961f, entries=150, sequenceid=229, filesize=11.9 K 2024-11-15T22:15:41,445 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~364.10 KB/372840, currentSize=53.67 KB/54960 for 0cf44e7263c9528c718ed800ce4e85e9 in 335ms, sequenceid=229, compaction requested=false 2024-11-15T22:15:41,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:41,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:41,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-15T22:15:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-15T22:15:41,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-15T22:15:41,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3260 sec 2024-11-15T22:15:41,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3550 sec 2024-11-15T22:15:41,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T22:15:41,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:41,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:41,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:41,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:41,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:41,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:41,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/2ff12a31af0749129dc5f7e14e1d144a is 50, key is test_row_0/A:col10/1731708941331/Put/seqid=0 2024-11-15T22:15:41,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741882_1058 (size=16931) 2024-11-15T22:15:41,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/2ff12a31af0749129dc5f7e14e1d144a 2024-11-15T22:15:41,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/6a22f8993d734064b5019c91106ebac4 is 50, key is test_row_0/B:col10/1731708941331/Put/seqid=0 2024-11-15T22:15:41,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709001529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:41,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:41,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709001531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:41,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:41,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709001537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:41,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741883_1059 (size=12151) 2024-11-15T22:15:41,640 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/204a062f4dc3469cb3a32f6d5cccd8c5 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/204a062f4dc3469cb3a32f6d5cccd8c5 2024-11-15T22:15:41,655 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/B of 0cf44e7263c9528c718ed800ce4e85e9 into 204a062f4dc3469cb3a32f6d5cccd8c5(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:41,655 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:41,655 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/B, priority=12, startTime=1731708941022; duration=0sec 2024-11-15T22:15:41,655 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:41,655 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:B 2024-11-15T22:15:41,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709001652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:41,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709001652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:41,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709001652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:41,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:41,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709001871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:41,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709001872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:41,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709001872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:41,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/6a22f8993d734064b5019c91106ebac4 2024-11-15T22:15:41,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/4894ee26390144ca9c79f05f49294822 is 50, key is test_row_0/C:col10/1731708941331/Put/seqid=0 2024-11-15T22:15:41,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741884_1060 (size=12151) 2024-11-15T22:15:42,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709002191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:42,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:42,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709002191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:42,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709002192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T22:15:42,272 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:15:42,272 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:15:42,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-15T22:15:42,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T22:15:42,276 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:15:42,277 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:15:42,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:15:42,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/4894ee26390144ca9c79f05f49294822 2024-11-15T22:15:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T22:15:42,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/2ff12a31af0749129dc5f7e14e1d144a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2ff12a31af0749129dc5f7e14e1d144a 2024-11-15T22:15:42,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2ff12a31af0749129dc5f7e14e1d144a, entries=250, sequenceid=244, filesize=16.5 K 2024-11-15T22:15:42,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/6a22f8993d734064b5019c91106ebac4 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/6a22f8993d734064b5019c91106ebac4 2024-11-15T22:15:42,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/6a22f8993d734064b5019c91106ebac4, entries=150, sequenceid=244, filesize=11.9 K 2024-11-15T22:15:42,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/4894ee26390144ca9c79f05f49294822 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/4894ee26390144ca9c79f05f49294822 2024-11-15T22:15:42,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/4894ee26390144ca9c79f05f49294822, entries=150, sequenceid=244, filesize=11.9 K 2024-11-15T22:15:42,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0cf44e7263c9528c718ed800ce4e85e9 in 953ms, sequenceid=244, compaction requested=true 2024-11-15T22:15:42,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:42,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:15:42,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:42,419 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:42,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:15:42,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:42,419 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:42,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:15:42,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:42,420 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:42,420 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:42,420 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/A is initiating minor compaction (all files) 2024-11-15T22:15:42,420 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/B is initiating minor compaction (all files) 2024-11-15T22:15:42,420 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/A in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:42,420 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/B in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:42,420 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/24e51f5feac6412ea6c16b37cca0ae0d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/12a6c4989ff941f5a19c1ea857e4a087, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2ff12a31af0749129dc5f7e14e1d144a] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=40.7 K 2024-11-15T22:15:42,420 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/204a062f4dc3469cb3a32f6d5cccd8c5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d5b8c99767ba44df8c405f02951f0d2e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/6a22f8993d734064b5019c91106ebac4] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=36.0 K 2024-11-15T22:15:42,421 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24e51f5feac6412ea6c16b37cca0ae0d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731708939888 2024-11-15T22:15:42,421 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 204a062f4dc3469cb3a32f6d5cccd8c5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731708939888 2024-11-15T22:15:42,421 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12a6c4989ff941f5a19c1ea857e4a087, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731708940077 2024-11-15T22:15:42,421 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d5b8c99767ba44df8c405f02951f0d2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731708940077 2024-11-15T22:15:42,422 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ff12a31af0749129dc5f7e14e1d144a, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731708941331 2024-11-15T22:15:42,422 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a22f8993d734064b5019c91106ebac4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731708941331 2024-11-15T22:15:42,433 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#B#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:42,433 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:42,434 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/758bd910162140dbb6029e16eabfe585 is 50, key is test_row_0/B:col10/1731708941331/Put/seqid=0 2024-11-15T22:15:42,434 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/73a6269eb675468ea8baeadd059411aa is 50, key is test_row_0/A:col10/1731708941331/Put/seqid=0 2024-11-15T22:15:42,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T22:15:42,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:42,436 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T22:15:42,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:42,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:42,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:42,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:42,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:42,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:42,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/f0c76803852647539b3e79f407f59f09 is 50, key is test_row_0/A:col10/1731708941534/Put/seqid=0 2024-11-15T22:15:42,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741885_1061 (size=12697) 2024-11-15T22:15:42,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741886_1062 (size=12697) 2024-11-15T22:15:42,480 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/758bd910162140dbb6029e16eabfe585 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/758bd910162140dbb6029e16eabfe585 2024-11-15T22:15:42,494 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/B of 0cf44e7263c9528c718ed800ce4e85e9 into 758bd910162140dbb6029e16eabfe585(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:42,494 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:42,494 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/B, priority=13, startTime=1731708942419; duration=0sec 2024-11-15T22:15:42,494 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:42,494 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:B 2024-11-15T22:15:42,494 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:42,496 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:42,496 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/C is initiating minor compaction (all files) 2024-11-15T22:15:42,496 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/C in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:42,496 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/426ad5360ce44327bd5e132373817bae, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/908d88dba55a4e36828e081c1126961f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/4894ee26390144ca9c79f05f49294822] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=36.0 K 2024-11-15T22:15:42,499 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 426ad5360ce44327bd5e132373817bae, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731708939888 2024-11-15T22:15:42,500 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 908d88dba55a4e36828e081c1126961f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731708940077 2024-11-15T22:15:42,501 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 4894ee26390144ca9c79f05f49294822, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731708941331 2024-11-15T22:15:42,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741887_1063 (size=12301) 2024-11-15T22:15:42,513 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/f0c76803852647539b3e79f407f59f09 2024-11-15T22:15:42,547 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:42,548 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/dadb0f72baad4500ab1736f3d7987f68 is 50, key is test_row_0/C:col10/1731708941331/Put/seqid=0 2024-11-15T22:15:42,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/33feb232565d47d18cb2d55226e8028d is 50, key is test_row_0/B:col10/1731708941534/Put/seqid=0 2024-11-15T22:15:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T22:15:42,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741888_1064 (size=12697) 2024-11-15T22:15:42,624 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/dadb0f72baad4500ab1736f3d7987f68 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/dadb0f72baad4500ab1736f3d7987f68 2024-11-15T22:15:42,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741889_1065 (size=12301) 2024-11-15T22:15:42,638 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/33feb232565d47d18cb2d55226e8028d 2024-11-15T22:15:42,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/c2eab01d88f34b9f93b32302caccf267 is 50, key is test_row_0/C:col10/1731708941534/Put/seqid=0 2024-11-15T22:15:42,657 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/C of 0cf44e7263c9528c718ed800ce4e85e9 into dadb0f72baad4500ab1736f3d7987f68(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:42,657 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:42,657 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/C, priority=13, startTime=1731708942419; duration=0sec 2024-11-15T22:15:42,657 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:42,657 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:C 2024-11-15T22:15:42,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741890_1066 (size=12301) 2024-11-15T22:15:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:42,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:42,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709002726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:42,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709002729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:42,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:42,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709002730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:42,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:42,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731709002822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:42,824 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in 0cf44e7263c9528c718ed800ce4e85e9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:15:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:42,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731709002831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:42,837 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in 0cf44e7263c9528c718ed800ce4e85e9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:15:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:42,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709002842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:42,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:42,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709002842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709002842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:42,891 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/73a6269eb675468ea8baeadd059411aa as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/73a6269eb675468ea8baeadd059411aa 2024-11-15T22:15:42,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T22:15:42,903 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/A of 0cf44e7263c9528c718ed800ce4e85e9 into 73a6269eb675468ea8baeadd059411aa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:42,903 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:42,903 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/A, priority=13, startTime=1731708942418; duration=0sec 2024-11-15T22:15:42,903 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:42,903 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:A 2024-11-15T22:15:43,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709003054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:43,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709003054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:43,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709003054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:43,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:43,095 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/c2eab01d88f34b9f93b32302caccf267 2024-11-15T22:15:43,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/f0c76803852647539b3e79f407f59f09 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f0c76803852647539b3e79f407f59f09 2024-11-15T22:15:43,128 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f0c76803852647539b3e79f407f59f09, entries=150, sequenceid=264, filesize=12.0 K 2024-11-15T22:15:43,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/33feb232565d47d18cb2d55226e8028d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/33feb232565d47d18cb2d55226e8028d 2024-11-15T22:15:43,143 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/33feb232565d47d18cb2d55226e8028d, entries=150, sequenceid=264, filesize=12.0 K 2024-11-15T22:15:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/c2eab01d88f34b9f93b32302caccf267 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/c2eab01d88f34b9f93b32302caccf267 2024-11-15T22:15:43,157 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/c2eab01d88f34b9f93b32302caccf267, entries=150, sequenceid=264, filesize=12.0 K 2024-11-15T22:15:43,158 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 0cf44e7263c9528c718ed800ce4e85e9 in 723ms, sequenceid=264, compaction requested=false 2024-11-15T22:15:43,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:43,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:43,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-15T22:15:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-15T22:15:43,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-15T22:15:43,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 883 msec 2024-11-15T22:15:43,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 893 msec 2024-11-15T22:15:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:43,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T22:15:43,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:43,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:43,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:43,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:43,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:43,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:43,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/f8c0df7acbb24349a8b81b4a624be476 is 50, key is test_row_0/A:col10/1731708942724/Put/seqid=0 2024-11-15T22:15:43,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741891_1067 (size=17181) 2024-11-15T22:15:43,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/f8c0df7acbb24349a8b81b4a624be476 2024-11-15T22:15:43,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709003394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:43,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709003396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:43,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:43,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709003403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:43,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/d80c62a3b00444bc85e96be1c08f69de is 50, key is test_row_0/B:col10/1731708942724/Put/seqid=0 2024-11-15T22:15:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T22:15:43,412 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:15:43,413 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:15:43,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-15T22:15:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T22:15:43,417 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:15:43,418 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:15:43,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:15:43,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741892_1068 (size=12301) 2024-11-15T22:15:43,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/d80c62a3b00444bc85e96be1c08f69de 2024-11-15T22:15:43,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/a198b6d34545475b93ce63c93ff7c0bf is 50, key is test_row_0/C:col10/1731708942724/Put/seqid=0 2024-11-15T22:15:43,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741893_1069 (size=12301) 2024-11-15T22:15:43,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/a198b6d34545475b93ce63c93ff7c0bf 2024-11-15T22:15:43,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/f8c0df7acbb24349a8b81b4a624be476 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f8c0df7acbb24349a8b81b4a624be476 2024-11-15T22:15:43,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:43,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709003511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:43,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709003511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709003512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:43,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f8c0df7acbb24349a8b81b4a624be476, entries=250, sequenceid=283, filesize=16.8 K 2024-11-15T22:15:43,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/d80c62a3b00444bc85e96be1c08f69de as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d80c62a3b00444bc85e96be1c08f69de 2024-11-15T22:15:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T22:15:43,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d80c62a3b00444bc85e96be1c08f69de, entries=150, sequenceid=283, filesize=12.0 K 2024-11-15T22:15:43,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/a198b6d34545475b93ce63c93ff7c0bf as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a198b6d34545475b93ce63c93ff7c0bf 2024-11-15T22:15:43,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a198b6d34545475b93ce63c93ff7c0bf, entries=150, sequenceid=283, filesize=12.0 K 2024-11-15T22:15:43,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 0cf44e7263c9528c718ed800ce4e85e9 in 178ms, sequenceid=283, compaction requested=true 2024-11-15T22:15:43,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:43,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:15:43,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:43,543 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:43,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:15:43,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:43,543 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:43,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:15:43,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:43,544 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42179 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:43,544 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/A is initiating minor compaction (all files) 2024-11-15T22:15:43,544 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/A in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:43,545 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/73a6269eb675468ea8baeadd059411aa, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f0c76803852647539b3e79f407f59f09, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f8c0df7acbb24349a8b81b4a624be476] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=41.2 K 2024-11-15T22:15:43,545 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:43,545 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/B is initiating minor compaction (all files) 2024-11-15T22:15:43,545 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/B in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:43,545 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/758bd910162140dbb6029e16eabfe585, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/33feb232565d47d18cb2d55226e8028d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d80c62a3b00444bc85e96be1c08f69de] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=36.4 K 2024-11-15T22:15:43,546 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 758bd910162140dbb6029e16eabfe585, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731708941331 2024-11-15T22:15:43,546 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73a6269eb675468ea8baeadd059411aa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731708941331 2024-11-15T22:15:43,546 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 33feb232565d47d18cb2d55226e8028d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731708941515 2024-11-15T22:15:43,546 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0c76803852647539b3e79f407f59f09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731708941515 2024-11-15T22:15:43,547 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d80c62a3b00444bc85e96be1c08f69de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731708942724 2024-11-15T22:15:43,547 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8c0df7acbb24349a8b81b4a624be476, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731708942710 2024-11-15T22:15:43,563 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:43,564 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/af9f5d56fefb4992bec621232bfe67a5 is 50, key is test_row_0/B:col10/1731708942724/Put/seqid=0 2024-11-15T22:15:43,564 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#A#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:43,565 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/0ed8ef10e73a4c5bbe632fa93a423c88 is 50, key is test_row_0/A:col10/1731708942724/Put/seqid=0 2024-11-15T22:15:43,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T22:15:43,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:43,575 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T22:15:43,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:43,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:43,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:43,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:43,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:43,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:43,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741894_1070 (size=12949) 2024-11-15T22:15:43,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/4c4cd510d6ce420a9b6f2b8084898787 is 50, key is test_row_0/A:col10/1731708943385/Put/seqid=0 2024-11-15T22:15:43,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741895_1071 (size=12949) 2024-11-15T22:15:43,634 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/af9f5d56fefb4992bec621232bfe67a5 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/af9f5d56fefb4992bec621232bfe67a5 2024-11-15T22:15:43,642 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/B of 0cf44e7263c9528c718ed800ce4e85e9 into af9f5d56fefb4992bec621232bfe67a5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:43,642 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:43,642 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/B, priority=13, startTime=1731708943543; duration=0sec 2024-11-15T22:15:43,642 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:43,642 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:B 2024-11-15T22:15:43,642 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:43,643 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:43,644 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/C is initiating minor compaction (all files) 2024-11-15T22:15:43,644 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/C in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:43,644 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/dadb0f72baad4500ab1736f3d7987f68, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/c2eab01d88f34b9f93b32302caccf267, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a198b6d34545475b93ce63c93ff7c0bf] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=36.4 K 2024-11-15T22:15:43,644 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting dadb0f72baad4500ab1736f3d7987f68, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731708941331 2024-11-15T22:15:43,645 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c2eab01d88f34b9f93b32302caccf267, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731708941515 2024-11-15T22:15:43,645 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting a198b6d34545475b93ce63c93ff7c0bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731708942724 2024-11-15T22:15:43,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741896_1072 (size=12301) 2024-11-15T22:15:43,649 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/4c4cd510d6ce420a9b6f2b8084898787 2024-11-15T22:15:43,661 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#C#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:43,662 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/a49048b6877646cbb71b3d58c4833f99 is 50, key is test_row_0/C:col10/1731708942724/Put/seqid=0 2024-11-15T22:15:43,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/4e5a4612dd1c4ccaadde83d11358f748 is 50, key is test_row_0/B:col10/1731708943385/Put/seqid=0 2024-11-15T22:15:43,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741897_1073 (size=12949) 2024-11-15T22:15:43,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741898_1074 (size=12301) 2024-11-15T22:15:43,690 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/4e5a4612dd1c4ccaadde83d11358f748 2024-11-15T22:15:43,695 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/a49048b6877646cbb71b3d58c4833f99 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a49048b6877646cbb71b3d58c4833f99 2024-11-15T22:15:43,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/62fa340ff3b349a88f573672fdb99b82 is 50, key is test_row_0/C:col10/1731708943385/Put/seqid=0 2024-11-15T22:15:43,716 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/C of 0cf44e7263c9528c718ed800ce4e85e9 into a49048b6877646cbb71b3d58c4833f99(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:43,716 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:43,716 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/C, priority=13, startTime=1731708943543; duration=0sec 2024-11-15T22:15:43,716 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:43,716 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:C 2024-11-15T22:15:43,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741899_1075 (size=12301) 2024-11-15T22:15:43,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T22:15:43,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709003756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:43,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709003759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709003761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:43,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709003871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:43,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709003871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709003872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:44,013 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/0ed8ef10e73a4c5bbe632fa93a423c88 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/0ed8ef10e73a4c5bbe632fa93a423c88 2024-11-15T22:15:44,021 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/A of 0cf44e7263c9528c718ed800ce4e85e9 into 0ed8ef10e73a4c5bbe632fa93a423c88(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:44,021 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:44,021 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/A, priority=13, startTime=1731708943542; duration=0sec 2024-11-15T22:15:44,021 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:44,021 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:A 2024-11-15T22:15:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T22:15:44,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709004082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:44,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709004082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:44,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709004083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:44,119 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/62fa340ff3b349a88f573672fdb99b82 2024-11-15T22:15:44,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/4c4cd510d6ce420a9b6f2b8084898787 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/4c4cd510d6ce420a9b6f2b8084898787 2024-11-15T22:15:44,147 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/4c4cd510d6ce420a9b6f2b8084898787, entries=150, sequenceid=299, filesize=12.0 K 2024-11-15T22:15:44,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/4e5a4612dd1c4ccaadde83d11358f748 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/4e5a4612dd1c4ccaadde83d11358f748 2024-11-15T22:15:44,162 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/4e5a4612dd1c4ccaadde83d11358f748, entries=150, sequenceid=299, filesize=12.0 K 2024-11-15T22:15:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/62fa340ff3b349a88f573672fdb99b82 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/62fa340ff3b349a88f573672fdb99b82 2024-11-15T22:15:44,175 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/62fa340ff3b349a88f573672fdb99b82, entries=150, sequenceid=299, filesize=12.0 K 2024-11-15T22:15:44,177 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 0cf44e7263c9528c718ed800ce4e85e9 in 602ms, sequenceid=299, compaction requested=false 2024-11-15T22:15:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-15T22:15:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-15T22:15:44,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-15T22:15:44,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 762 msec 2024-11-15T22:15:44,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 776 msec 2024-11-15T22:15:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:44,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T22:15:44,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:44,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:44,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:44,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:44,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:44,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:44,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/6bf11aece7154541b89e821c123791fc is 50, key is test_row_0/A:col10/1731708944392/Put/seqid=0 2024-11-15T22:15:44,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741900_1076 (size=12301) 2024-11-15T22:15:44,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709004430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:44,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709004432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709004432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:44,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709004542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:44,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:44,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709004542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709004542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T22:15:44,552 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:15:44,552 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:15:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-15T22:15:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T22:15:44,557 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:15:44,558 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:15:44,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:15:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T22:15:44,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T22:15:44,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:44,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:44,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:44,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:44,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:44,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709004752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:44,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:44,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:44,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709004752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:44,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:44,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:44,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709004752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:44,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/6bf11aece7154541b89e821c123791fc 2024-11-15T22:15:44,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/6eb79b34508a4e3ba7cf34b9c05d3f33 is 50, key is test_row_0/B:col10/1731708944392/Put/seqid=0 2024-11-15T22:15:44,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741901_1077 (size=12301) 2024-11-15T22:15:44,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T22:15:44,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:44,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:44,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:44,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T22:15:44,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:44,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T22:15:45,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:45,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:45,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:45,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:45,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709005062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:45,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709005062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:45,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709005062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T22:15:45,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T22:15:45,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:45,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:45,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:45,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/6eb79b34508a4e3ba7cf34b9c05d3f33 2024-11-15T22:15:45,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/8fa9626afc2a4656998f8f0603266fed is 50, key is test_row_0/C:col10/1731708944392/Put/seqid=0 2024-11-15T22:15:45,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741902_1078 (size=12301) 2024-11-15T22:15:45,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T22:15:45,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:45,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:45,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:45,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T22:15:45,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:45,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:45,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:45,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709005571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:45,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:45,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709005571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709005571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:45,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T22:15:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:45,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:45,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/8fa9626afc2a4656998f8f0603266fed 2024-11-15T22:15:45,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/6bf11aece7154541b89e821c123791fc as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/6bf11aece7154541b89e821c123791fc 2024-11-15T22:15:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T22:15:45,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/6bf11aece7154541b89e821c123791fc, entries=150, sequenceid=320, filesize=12.0 K 2024-11-15T22:15:45,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/6eb79b34508a4e3ba7cf34b9c05d3f33 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/6eb79b34508a4e3ba7cf34b9c05d3f33 2024-11-15T22:15:45,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/6eb79b34508a4e3ba7cf34b9c05d3f33, entries=150, sequenceid=320, filesize=12.0 K 2024-11-15T22:15:45,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/8fa9626afc2a4656998f8f0603266fed as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8fa9626afc2a4656998f8f0603266fed 2024-11-15T22:15:45,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8fa9626afc2a4656998f8f0603266fed, entries=150, sequenceid=320, filesize=12.0 K 2024-11-15T22:15:45,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 0cf44e7263c9528c718ed800ce4e85e9 in 1322ms, sequenceid=320, compaction requested=true 2024-11-15T22:15:45,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:45,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:15:45,717 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:45,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:45,717 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:45,718 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:45,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:15:45,718 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/A is initiating minor compaction (all files) 2024-11-15T22:15:45,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:45,719 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:45,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:15:45,720 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/B is initiating minor compaction (all files) 2024-11-15T22:15:45,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:45,720 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/B in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:45,720 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/af9f5d56fefb4992bec621232bfe67a5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/4e5a4612dd1c4ccaadde83d11358f748, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/6eb79b34508a4e3ba7cf34b9c05d3f33] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=36.7 K 2024-11-15T22:15:45,718 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/A in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:45,720 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/0ed8ef10e73a4c5bbe632fa93a423c88, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/4c4cd510d6ce420a9b6f2b8084898787, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/6bf11aece7154541b89e821c123791fc] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=36.7 K 2024-11-15T22:15:45,721 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting af9f5d56fefb4992bec621232bfe67a5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731708942724 2024-11-15T22:15:45,721 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ed8ef10e73a4c5bbe632fa93a423c88, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731708942724 2024-11-15T22:15:45,721 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e5a4612dd1c4ccaadde83d11358f748, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731708943385 2024-11-15T22:15:45,721 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c4cd510d6ce420a9b6f2b8084898787, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731708943385 2024-11-15T22:15:45,721 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb79b34508a4e3ba7cf34b9c05d3f33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731708943748 2024-11-15T22:15:45,721 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bf11aece7154541b89e821c123791fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731708943748 2024-11-15T22:15:45,741 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#B#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:45,741 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/0306fc712ae242c68fce5f7ba38e88a3 is 50, key is test_row_0/B:col10/1731708944392/Put/seqid=0 2024-11-15T22:15:45,746 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:45,747 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/d130f2c08b9e4a08992ce4fee866d24c is 50, key is test_row_0/A:col10/1731708944392/Put/seqid=0 2024-11-15T22:15:45,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741903_1079 (size=13051) 2024-11-15T22:15:45,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741904_1080 (size=13051) 2024-11-15T22:15:45,773 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/d130f2c08b9e4a08992ce4fee866d24c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/d130f2c08b9e4a08992ce4fee866d24c 2024-11-15T22:15:45,785 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/A of 0cf44e7263c9528c718ed800ce4e85e9 into d130f2c08b9e4a08992ce4fee866d24c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:45,786 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:45,786 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/A, priority=13, startTime=1731708945717; duration=0sec 2024-11-15T22:15:45,786 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:45,786 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:A 2024-11-15T22:15:45,787 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:45,788 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:45,788 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/C is initiating minor compaction (all files) 2024-11-15T22:15:45,788 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/C in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:45,788 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a49048b6877646cbb71b3d58c4833f99, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/62fa340ff3b349a88f573672fdb99b82, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8fa9626afc2a4656998f8f0603266fed] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=36.7 K 2024-11-15T22:15:45,789 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting a49048b6877646cbb71b3d58c4833f99, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731708942724 2024-11-15T22:15:45,789 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62fa340ff3b349a88f573672fdb99b82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731708943385 2024-11-15T22:15:45,789 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fa9626afc2a4656998f8f0603266fed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731708943748 2024-11-15T22:15:45,800 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:45,800 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/a8e0a27d87a846ecb642f79e59006d9e is 50, key is test_row_0/C:col10/1731708944392/Put/seqid=0 2024-11-15T22:15:45,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741905_1081 (size=13051) 2024-11-15T22:15:45,816 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/a8e0a27d87a846ecb642f79e59006d9e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a8e0a27d87a846ecb642f79e59006d9e 2024-11-15T22:15:45,824 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/C of 0cf44e7263c9528c718ed800ce4e85e9 into a8e0a27d87a846ecb642f79e59006d9e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:45,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T22:15:45,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:45,825 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T22:15:45,824 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:45,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:45,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:45,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:45,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:45,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:45,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:45,827 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/C, priority=13, startTime=1731708945720; duration=0sec 2024-11-15T22:15:45,827 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:45,827 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:C 2024-11-15T22:15:45,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/72429b7d77a64553b89a3070dfcd0d93 is 50, key is test_row_0/A:col10/1731708944422/Put/seqid=0 2024-11-15T22:15:45,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741906_1082 (size=12301) 2024-11-15T22:15:46,169 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/0306fc712ae242c68fce5f7ba38e88a3 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/0306fc712ae242c68fce5f7ba38e88a3 2024-11-15T22:15:46,184 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/B of 0cf44e7263c9528c718ed800ce4e85e9 into 0306fc712ae242c68fce5f7ba38e88a3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:46,184 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:46,184 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/B, priority=13, startTime=1731708945717; duration=0sec 2024-11-15T22:15:46,185 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:46,185 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:B 2024-11-15T22:15:46,242 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/72429b7d77a64553b89a3070dfcd0d93 2024-11-15T22:15:46,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/3a8e078fec6f4f1d86684770569f81d3 is 50, key is test_row_0/B:col10/1731708944422/Put/seqid=0 2024-11-15T22:15:46,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741907_1083 (size=12301) 2024-11-15T22:15:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:46,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:46,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709006597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:46,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709006599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:46,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709006599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:46,657 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/3a8e078fec6f4f1d86684770569f81d3 2024-11-15T22:15:46,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/7362289961fb4d1789f3bd00e58c58a7 is 50, key is test_row_0/C:col10/1731708944422/Put/seqid=0 2024-11-15T22:15:46,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741908_1084 (size=12301) 2024-11-15T22:15:46,673 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/7362289961fb4d1789f3bd00e58c58a7 2024-11-15T22:15:46,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/72429b7d77a64553b89a3070dfcd0d93 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/72429b7d77a64553b89a3070dfcd0d93 2024-11-15T22:15:46,683 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/72429b7d77a64553b89a3070dfcd0d93, entries=150, sequenceid=337, filesize=12.0 K 2024-11-15T22:15:46,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/3a8e078fec6f4f1d86684770569f81d3 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/3a8e078fec6f4f1d86684770569f81d3 2024-11-15T22:15:46,691 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/3a8e078fec6f4f1d86684770569f81d3, entries=150, sequenceid=337, filesize=12.0 K 2024-11-15T22:15:46,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/7362289961fb4d1789f3bd00e58c58a7 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7362289961fb4d1789f3bd00e58c58a7 2024-11-15T22:15:46,698 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7362289961fb4d1789f3bd00e58c58a7, entries=150, sequenceid=337, filesize=12.0 K 2024-11-15T22:15:46,700 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0cf44e7263c9528c718ed800ce4e85e9 in 874ms, sequenceid=337, compaction requested=false 2024-11-15T22:15:46,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:46,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:46,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-15T22:15:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-15T22:15:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T22:15:46,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-15T22:15:46,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1440 sec 2024-11-15T22:15:46,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.1570 sec 2024-11-15T22:15:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:46,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T22:15:46,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:46,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:46,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:46,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:46,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:46,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:46,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/4b9515d179a44795bef081a7c96bd59e is 50, key is test_row_0/A:col10/1731708946593/Put/seqid=0 2024-11-15T22:15:46,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741909_1085 (size=17181) 2024-11-15T22:15:46,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709006748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:46,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709006749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:46,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709006753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:46,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/4b9515d179a44795bef081a7c96bd59e 2024-11-15T22:15:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:46,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/99bfbd92cca84d768f5f352db8174a10 is 50, key is test_row_0/B:col10/1731708946593/Put/seqid=0 2024-11-15T22:15:46,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741910_1086 (size=12301) 2024-11-15T22:15:46,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:46,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38298 deadline: 1731709006862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:46,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:46,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38260 deadline: 1731709006862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:46,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:46,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709006863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:46,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:46,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709006863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:46,863 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in 0cf44e7263c9528c718ed800ce4e85e9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8300 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:15:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709006863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:46,866 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in 0cf44e7263c9528c718ed800ce4e85e9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8300 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:15:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:47,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709007071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:47,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709007071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:47,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709007071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:47,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:47,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/99bfbd92cca84d768f5f352db8174a10 2024-11-15T22:15:47,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/fd2e0c8683ba45ab9ca92350eec54147 is 50, key is test_row_0/C:col10/1731708946593/Put/seqid=0 2024-11-15T22:15:47,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741911_1087 (size=12301) 2024-11-15T22:15:47,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709007381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:47,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709007381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709007382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:47,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/fd2e0c8683ba45ab9ca92350eec54147 2024-11-15T22:15:47,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/4b9515d179a44795bef081a7c96bd59e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/4b9515d179a44795bef081a7c96bd59e 2024-11-15T22:15:47,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/4b9515d179a44795bef081a7c96bd59e, entries=250, sequenceid=357, filesize=16.8 K 2024-11-15T22:15:47,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/99bfbd92cca84d768f5f352db8174a10 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/99bfbd92cca84d768f5f352db8174a10 2024-11-15T22:15:47,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/99bfbd92cca84d768f5f352db8174a10, entries=150, sequenceid=357, filesize=12.0 K 2024-11-15T22:15:47,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/fd2e0c8683ba45ab9ca92350eec54147 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/fd2e0c8683ba45ab9ca92350eec54147 2024-11-15T22:15:47,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/fd2e0c8683ba45ab9ca92350eec54147, entries=150, sequenceid=357, filesize=12.0 K 2024-11-15T22:15:47,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 0cf44e7263c9528c718ed800ce4e85e9 in 908ms, sequenceid=357, compaction requested=true 2024-11-15T22:15:47,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:47,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:15:47,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:47,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:15:47,623 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:47,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:47,623 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:47,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:15:47,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:47,624 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:47,624 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:47,624 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/B is initiating minor compaction (all files) 2024-11-15T22:15:47,624 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/A is initiating minor compaction (all files) 2024-11-15T22:15:47,625 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/B in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:47,625 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/A in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:47,625 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/d130f2c08b9e4a08992ce4fee866d24c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/72429b7d77a64553b89a3070dfcd0d93, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/4b9515d179a44795bef081a7c96bd59e] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=41.5 K 2024-11-15T22:15:47,625 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/0306fc712ae242c68fce5f7ba38e88a3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/3a8e078fec6f4f1d86684770569f81d3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/99bfbd92cca84d768f5f352db8174a10] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=36.8 K 2024-11-15T22:15:47,625 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 0306fc712ae242c68fce5f7ba38e88a3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731708943748 2024-11-15T22:15:47,625 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting d130f2c08b9e4a08992ce4fee866d24c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731708943748 2024-11-15T22:15:47,626 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72429b7d77a64553b89a3070dfcd0d93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731708944422 2024-11-15T22:15:47,626 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a8e078fec6f4f1d86684770569f81d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731708944422 2024-11-15T22:15:47,626 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b9515d179a44795bef081a7c96bd59e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731708946591 2024-11-15T22:15:47,626 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 99bfbd92cca84d768f5f352db8174a10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731708946593 2024-11-15T22:15:47,639 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:47,640 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/43b8d8e66d784ef8b31d0325a53ac21d is 50, key is test_row_0/A:col10/1731708946593/Put/seqid=0 2024-11-15T22:15:47,644 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:47,645 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/ab0a275e2ad546c0a778b319e8af4ec3 is 50, key is test_row_0/B:col10/1731708946593/Put/seqid=0 2024-11-15T22:15:47,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741912_1088 (size=13153) 2024-11-15T22:15:47,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741913_1089 (size=13153) 2024-11-15T22:15:47,683 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/43b8d8e66d784ef8b31d0325a53ac21d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/43b8d8e66d784ef8b31d0325a53ac21d 2024-11-15T22:15:47,696 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/ab0a275e2ad546c0a778b319e8af4ec3 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/ab0a275e2ad546c0a778b319e8af4ec3 2024-11-15T22:15:47,699 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/A of 0cf44e7263c9528c718ed800ce4e85e9 into 43b8d8e66d784ef8b31d0325a53ac21d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:47,699 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:47,699 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/A, priority=13, startTime=1731708947623; duration=0sec 2024-11-15T22:15:47,701 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:47,701 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:A 2024-11-15T22:15:47,703 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:47,708 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:47,708 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/C is initiating minor compaction (all files) 2024-11-15T22:15:47,708 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/C in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:47,708 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a8e0a27d87a846ecb642f79e59006d9e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7362289961fb4d1789f3bd00e58c58a7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/fd2e0c8683ba45ab9ca92350eec54147] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=36.8 K 2024-11-15T22:15:47,709 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8e0a27d87a846ecb642f79e59006d9e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731708943748 2024-11-15T22:15:47,709 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7362289961fb4d1789f3bd00e58c58a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731708944422 2024-11-15T22:15:47,710 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd2e0c8683ba45ab9ca92350eec54147, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731708946593 2024-11-15T22:15:47,715 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/B of 0cf44e7263c9528c718ed800ce4e85e9 into ab0a275e2ad546c0a778b319e8af4ec3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:47,715 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:47,715 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/B, priority=13, startTime=1731708947623; duration=0sec 2024-11-15T22:15:47,716 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:47,716 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:B 2024-11-15T22:15:47,732 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#C#compaction#77 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:47,733 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/7f8d5058ec2e4a719d819a84b177f02d is 50, key is test_row_0/C:col10/1731708946593/Put/seqid=0 2024-11-15T22:15:47,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741914_1090 (size=13153) 2024-11-15T22:15:47,747 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/7f8d5058ec2e4a719d819a84b177f02d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7f8d5058ec2e4a719d819a84b177f02d 2024-11-15T22:15:47,758 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/C of 0cf44e7263c9528c718ed800ce4e85e9 into 7f8d5058ec2e4a719d819a84b177f02d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:47,758 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:47,758 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/C, priority=13, startTime=1731708947623; duration=0sec 2024-11-15T22:15:47,758 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:47,758 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:C 2024-11-15T22:15:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:47,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T22:15:47,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:47,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:47,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:47,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:47,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:47,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:47,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/913f5e0cf2a94d26b131e2af89bd4669 is 50, key is test_row_0/A:col10/1731708947892/Put/seqid=0 2024-11-15T22:15:47,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709007922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:47,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709007925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:47,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709007927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:47,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741915_1091 (size=12301) 2024-11-15T22:15:47,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/913f5e0cf2a94d26b131e2af89bd4669 2024-11-15T22:15:47,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/edbb2ebaed024d54bafeb9ef4213b6b3 is 50, key is test_row_0/B:col10/1731708947892/Put/seqid=0 2024-11-15T22:15:47,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741916_1092 (size=12301) 2024-11-15T22:15:47,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/edbb2ebaed024d54bafeb9ef4213b6b3 2024-11-15T22:15:47,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/91a93855915b459fb3c5c09d433c0d00 is 50, key is test_row_0/C:col10/1731708947892/Put/seqid=0 2024-11-15T22:15:47,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741917_1093 (size=12301) 2024-11-15T22:15:48,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/91a93855915b459fb3c5c09d433c0d00 2024-11-15T22:15:48,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/913f5e0cf2a94d26b131e2af89bd4669 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/913f5e0cf2a94d26b131e2af89bd4669 2024-11-15T22:15:48,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/913f5e0cf2a94d26b131e2af89bd4669, entries=150, sequenceid=375, filesize=12.0 K 2024-11-15T22:15:48,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/edbb2ebaed024d54bafeb9ef4213b6b3 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/edbb2ebaed024d54bafeb9ef4213b6b3 2024-11-15T22:15:48,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/edbb2ebaed024d54bafeb9ef4213b6b3, entries=150, sequenceid=375, filesize=12.0 K 2024-11-15T22:15:48,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709008032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:48,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/91a93855915b459fb3c5c09d433c0d00 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/91a93855915b459fb3c5c09d433c0d00 2024-11-15T22:15:48,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709008034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:48,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709008041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:48,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:48,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/91a93855915b459fb3c5c09d433c0d00, entries=150, sequenceid=375, filesize=12.0 K 2024-11-15T22:15:48,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0cf44e7263c9528c718ed800ce4e85e9 in 169ms, sequenceid=375, compaction requested=false 2024-11-15T22:15:48,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:48,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T22:15:48,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:48,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:48,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:48,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:48,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:48,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:48,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/5d5194cbafe24364972462ba763e4a62 is 50, key is test_row_0/A:col10/1731708947923/Put/seqid=0 2024-11-15T22:15:48,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741918_1094 (size=14741) 2024-11-15T22:15:48,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709008269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:48,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:48,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709008272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:48,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709008273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:48,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:48,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709008381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:48,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709008382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:48,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709008391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:48,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709008593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:48,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709008594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:48,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709008601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:48,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/5d5194cbafe24364972462ba763e4a62 2024-11-15T22:15:48,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/5c03e95ea2454ba1b0e444d36f3d9985 is 50, key is test_row_0/B:col10/1731708947923/Put/seqid=0 2024-11-15T22:15:48,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741919_1095 (size=12301) 2024-11-15T22:15:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T22:15:48,711 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:15:48,712 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:15:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-15T22:15:48,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T22:15:48,724 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:15:48,725 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:15:48,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:15:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T22:15:48,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T22:15:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:48,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:48,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709008902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:48,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709008902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:48,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709008911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:49,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T22:15:49,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:49,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:49,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:49,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:49,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T22:15:49,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/5c03e95ea2454ba1b0e444d36f3d9985 2024-11-15T22:15:49,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/f01a600344fb48179214c838c8654163 is 50, key is test_row_0/C:col10/1731708947923/Put/seqid=0 2024-11-15T22:15:49,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741920_1096 (size=12301) 2024-11-15T22:15:49,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T22:15:49,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:49,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:49,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:49,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:49,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:49,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:49,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T22:15:49,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T22:15:49,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:49,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:49,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:49,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:49,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:49,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:49,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709009422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:49,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709009422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:49,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709009422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:49,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/f01a600344fb48179214c838c8654163 2024-11-15T22:15:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/5d5194cbafe24364972462ba763e4a62 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/5d5194cbafe24364972462ba763e4a62 2024-11-15T22:15:49,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T22:15:49,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:49,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:49,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:49,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:49,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:49,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/5d5194cbafe24364972462ba763e4a62, entries=200, sequenceid=393, filesize=14.4 K 2024-11-15T22:15:49,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/5c03e95ea2454ba1b0e444d36f3d9985 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/5c03e95ea2454ba1b0e444d36f3d9985 2024-11-15T22:15:49,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/5c03e95ea2454ba1b0e444d36f3d9985, entries=150, sequenceid=393, filesize=12.0 K 2024-11-15T22:15:49,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/f01a600344fb48179214c838c8654163 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/f01a600344fb48179214c838c8654163 2024-11-15T22:15:49,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/f01a600344fb48179214c838c8654163, entries=150, sequenceid=393, filesize=12.0 K 2024-11-15T22:15:49,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0cf44e7263c9528c718ed800ce4e85e9 in 1374ms, sequenceid=393, compaction requested=true 2024-11-15T22:15:49,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:49,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:15:49,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:49,617 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:49,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:15:49,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:49,617 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:49,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:15:49,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:49,619 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:49,619 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:49,619 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/A is initiating minor compaction (all files) 2024-11-15T22:15:49,619 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/B is initiating minor compaction (all files) 2024-11-15T22:15:49,619 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/A in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:49,619 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/B in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:49,619 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/ab0a275e2ad546c0a778b319e8af4ec3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/edbb2ebaed024d54bafeb9ef4213b6b3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/5c03e95ea2454ba1b0e444d36f3d9985] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=36.9 K 2024-11-15T22:15:49,619 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/43b8d8e66d784ef8b31d0325a53ac21d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/913f5e0cf2a94d26b131e2af89bd4669, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/5d5194cbafe24364972462ba763e4a62] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=39.3 K 2024-11-15T22:15:49,620 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting ab0a275e2ad546c0a778b319e8af4ec3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731708946593 2024-11-15T22:15:49,620 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43b8d8e66d784ef8b31d0325a53ac21d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731708946593 2024-11-15T22:15:49,622 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 913f5e0cf2a94d26b131e2af89bd4669, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731708946745 2024-11-15T22:15:49,623 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting edbb2ebaed024d54bafeb9ef4213b6b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731708946745 2024-11-15T22:15:49,623 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d5194cbafe24364972462ba763e4a62, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731708947908 2024-11-15T22:15:49,623 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c03e95ea2454ba1b0e444d36f3d9985, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731708947923 2024-11-15T22:15:49,635 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:49,635 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/1f361bf5612e40ecb7ef243d60b0cbf6 is 50, key is test_row_0/A:col10/1731708947923/Put/seqid=0 2024-11-15T22:15:49,636 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#B#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:49,636 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/aae6f4a43f3249379f3bff06c16ae9c2 is 50, key is test_row_0/B:col10/1731708947923/Put/seqid=0 2024-11-15T22:15:49,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741921_1097 (size=13255) 2024-11-15T22:15:49,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741922_1098 (size=13255) 2024-11-15T22:15:49,662 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/1f361bf5612e40ecb7ef243d60b0cbf6 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/1f361bf5612e40ecb7ef243d60b0cbf6 2024-11-15T22:15:49,665 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/aae6f4a43f3249379f3bff06c16ae9c2 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/aae6f4a43f3249379f3bff06c16ae9c2 2024-11-15T22:15:49,672 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/A of 0cf44e7263c9528c718ed800ce4e85e9 into 1f361bf5612e40ecb7ef243d60b0cbf6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:49,672 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:49,672 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/A, priority=13, startTime=1731708949617; duration=0sec 2024-11-15T22:15:49,672 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:49,672 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:A 2024-11-15T22:15:49,672 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:49,674 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/B of 0cf44e7263c9528c718ed800ce4e85e9 into aae6f4a43f3249379f3bff06c16ae9c2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:49,674 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:49,674 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/B, priority=13, startTime=1731708949617; duration=0sec 2024-11-15T22:15:49,674 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:49,674 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:B 2024-11-15T22:15:49,675 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:49,675 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/C is initiating minor compaction (all files) 2024-11-15T22:15:49,675 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/C in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:49,675 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7f8d5058ec2e4a719d819a84b177f02d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/91a93855915b459fb3c5c09d433c0d00, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/f01a600344fb48179214c838c8654163] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=36.9 K 2024-11-15T22:15:49,676 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f8d5058ec2e4a719d819a84b177f02d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731708946593 2024-11-15T22:15:49,676 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91a93855915b459fb3c5c09d433c0d00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731708946745 2024-11-15T22:15:49,677 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting f01a600344fb48179214c838c8654163, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731708947923 2024-11-15T22:15:49,692 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#C#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:49,696 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/d27d1ecaee6f4719a3ee0b4e5fb54dcf is 50, key is test_row_0/C:col10/1731708947923/Put/seqid=0 2024-11-15T22:15:49,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741923_1099 (size=13255) 2024-11-15T22:15:49,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T22:15:49,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:49,711 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T22:15:49,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:49,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:49,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:49,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:49,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:49,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:49,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/62e26eae80db4953a116e8f98ce33ee2 is 50, key is test_row_0/A:col10/1731708948271/Put/seqid=0 2024-11-15T22:15:49,725 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/d27d1ecaee6f4719a3ee0b4e5fb54dcf as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/d27d1ecaee6f4719a3ee0b4e5fb54dcf 2024-11-15T22:15:49,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741924_1100 (size=12301) 2024-11-15T22:15:49,729 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/62e26eae80db4953a116e8f98ce33ee2 2024-11-15T22:15:49,736 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/C of 0cf44e7263c9528c718ed800ce4e85e9 into d27d1ecaee6f4719a3ee0b4e5fb54dcf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:49,736 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:49,736 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/C, priority=13, startTime=1731708949617; duration=0sec 2024-11-15T22:15:49,737 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:49,737 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:C 2024-11-15T22:15:49,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/083b06be052547e2903c162537e1078c is 50, key is test_row_0/B:col10/1731708948271/Put/seqid=0 2024-11-15T22:15:49,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741925_1101 (size=12301) 2024-11-15T22:15:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T22:15:50,172 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/083b06be052547e2903c162537e1078c 2024-11-15T22:15:50,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/d51a89da0ea64d10aaf827143d3616c0 is 50, key is test_row_0/C:col10/1731708948271/Put/seqid=0 2024-11-15T22:15:50,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741926_1102 (size=12301) 2024-11-15T22:15:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:50,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:50,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709010509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:50,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:50,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:50,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709010517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:50,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:50,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709010517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:50,621 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/d51a89da0ea64d10aaf827143d3616c0 2024-11-15T22:15:50,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709010632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:50,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:50,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/62e26eae80db4953a116e8f98ce33ee2 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/62e26eae80db4953a116e8f98ce33ee2 2024-11-15T22:15:50,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709010644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:50,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:50,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709010643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:50,661 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/62e26eae80db4953a116e8f98ce33ee2, entries=150, sequenceid=409, filesize=12.0 K 2024-11-15T22:15:50,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/083b06be052547e2903c162537e1078c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/083b06be052547e2903c162537e1078c 2024-11-15T22:15:50,672 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/083b06be052547e2903c162537e1078c, entries=150, sequenceid=409, filesize=12.0 K 2024-11-15T22:15:50,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/d51a89da0ea64d10aaf827143d3616c0 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/d51a89da0ea64d10aaf827143d3616c0 2024-11-15T22:15:50,680 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/d51a89da0ea64d10aaf827143d3616c0, entries=150, sequenceid=409, filesize=12.0 K 2024-11-15T22:15:50,686 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0cf44e7263c9528c718ed800ce4e85e9 in 971ms, sequenceid=409, compaction requested=false 2024-11-15T22:15:50,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:50,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:50,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-15T22:15:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-15T22:15:50,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-15T22:15:50,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9640 sec 2024-11-15T22:15:50,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.9830 sec 2024-11-15T22:15:50,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T22:15:50,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:50,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:50,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:50,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:50,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:50,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:50,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/7ee8f8ef787b4b5ba297a118a2484be8 is 50, key is test_row_0/A:col10/1731708950506/Put/seqid=0 2024-11-15T22:15:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T22:15:50,872 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:15:50,873 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:15:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-15T22:15:50,876 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:15:50,879 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:15:50,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:15:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T22:15:50,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741927_1103 (size=14741) 2024-11-15T22:15:50,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709010936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:50,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709010946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:50,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:50,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709010957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:50,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:50,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T22:15:51,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:15:51,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:51,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:51,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:51,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709011061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:51,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:51,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:51,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709011083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709011083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:51,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:51,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:15:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:51,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T22:15:51,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709011283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:51,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/7ee8f8ef787b4b5ba297a118a2484be8 2024-11-15T22:15:51,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709011299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:51,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709011300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:51,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:51,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/9e0b21788a1047ea96d87de30563f6ac is 50, key is test_row_0/B:col10/1731708950506/Put/seqid=0 2024-11-15T22:15:51,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:15:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:51,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741928_1104 (size=12301) 2024-11-15T22:15:51,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/9e0b21788a1047ea96d87de30563f6ac 2024-11-15T22:15:51,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/8c17c6dbdb424e3281e2acb1b52ee4d7 is 50, key is test_row_0/C:col10/1731708950506/Put/seqid=0 2024-11-15T22:15:51,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741929_1105 (size=12301) 2024-11-15T22:15:51,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/8c17c6dbdb424e3281e2acb1b52ee4d7 2024-11-15T22:15:51,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/7ee8f8ef787b4b5ba297a118a2484be8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7ee8f8ef787b4b5ba297a118a2484be8 2024-11-15T22:15:51,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7ee8f8ef787b4b5ba297a118a2484be8, entries=200, sequenceid=429, filesize=14.4 K 2024-11-15T22:15:51,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/9e0b21788a1047ea96d87de30563f6ac as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/9e0b21788a1047ea96d87de30563f6ac 2024-11-15T22:15:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T22:15:51,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/9e0b21788a1047ea96d87de30563f6ac, entries=150, sequenceid=429, filesize=12.0 K 2024-11-15T22:15:51,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:15:51,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:51,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:51,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:51,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/8c17c6dbdb424e3281e2acb1b52ee4d7 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8c17c6dbdb424e3281e2acb1b52ee4d7 2024-11-15T22:15:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8c17c6dbdb424e3281e2acb1b52ee4d7, entries=150, sequenceid=429, filesize=12.0 K 2024-11-15T22:15:51,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 0cf44e7263c9528c718ed800ce4e85e9 in 699ms, sequenceid=429, compaction requested=true 2024-11-15T22:15:51,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:51,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:15:51,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:51,548 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:51,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:15:51,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:51,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:15:51,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:15:51,549 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:51,553 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:51,553 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:51,553 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/B is initiating minor compaction (all files) 2024-11-15T22:15:51,553 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/A is initiating minor compaction (all files) 2024-11-15T22:15:51,553 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/B in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:51,553 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/A in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:51,553 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/1f361bf5612e40ecb7ef243d60b0cbf6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/62e26eae80db4953a116e8f98ce33ee2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7ee8f8ef787b4b5ba297a118a2484be8] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=39.4 K 2024-11-15T22:15:51,553 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/aae6f4a43f3249379f3bff06c16ae9c2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/083b06be052547e2903c162537e1078c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/9e0b21788a1047ea96d87de30563f6ac] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=37.0 K 2024-11-15T22:15:51,554 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting aae6f4a43f3249379f3bff06c16ae9c2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731708947923 2024-11-15T22:15:51,554 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f361bf5612e40ecb7ef243d60b0cbf6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731708947923 2024-11-15T22:15:51,555 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 083b06be052547e2903c162537e1078c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731708948264 2024-11-15T22:15:51,555 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62e26eae80db4953a116e8f98ce33ee2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731708948264 2024-11-15T22:15:51,555 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e0b21788a1047ea96d87de30563f6ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731708950506 2024-11-15T22:15:51,556 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ee8f8ef787b4b5ba297a118a2484be8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731708950492 2024-11-15T22:15:51,592 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#A#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:51,593 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/a0975f2144704f86812c3e56e67fb2c2 is 50, key is test_row_0/A:col10/1731708950506/Put/seqid=0 2024-11-15T22:15:51,614 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#B#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:51,614 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/ff549231cfe849babd43e5155d393587 is 50, key is test_row_0/B:col10/1731708950506/Put/seqid=0 2024-11-15T22:15:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:51,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T22:15:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:51,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741930_1106 (size=13357) 2024-11-15T22:15:51,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/26c25acc05d94759995a243d4038d862 is 50, key is test_row_0/A:col10/1731708950940/Put/seqid=0 2024-11-15T22:15:51,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741931_1107 (size=13357) 2024-11-15T22:15:51,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:15:51,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:51,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:51,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:51,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,695 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/ff549231cfe849babd43e5155d393587 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/ff549231cfe849babd43e5155d393587 2024-11-15T22:15:51,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741932_1108 (size=12301) 2024-11-15T22:15:51,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/26c25acc05d94759995a243d4038d862 2024-11-15T22:15:51,705 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/B of 0cf44e7263c9528c718ed800ce4e85e9 into ff549231cfe849babd43e5155d393587(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:51,705 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:51,705 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/B, priority=13, startTime=1731708951548; duration=0sec 2024-11-15T22:15:51,706 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:51,706 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:B 2024-11-15T22:15:51,706 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:51,708 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:51,709 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/C is initiating minor compaction (all files) 2024-11-15T22:15:51,709 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/C in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:51,709 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/d27d1ecaee6f4719a3ee0b4e5fb54dcf, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/d51a89da0ea64d10aaf827143d3616c0, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8c17c6dbdb424e3281e2acb1b52ee4d7] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=37.0 K 2024-11-15T22:15:51,710 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d27d1ecaee6f4719a3ee0b4e5fb54dcf, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731708947923 2024-11-15T22:15:51,713 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d51a89da0ea64d10aaf827143d3616c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731708948264 2024-11-15T22:15:51,714 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c17c6dbdb424e3281e2acb1b52ee4d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731708950506 2024-11-15T22:15:51,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/c418b178568b4f32ad2d4debe8206568 is 50, key is test_row_0/B:col10/1731708950940/Put/seqid=0 2024-11-15T22:15:51,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709011710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:51,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709011717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:51,743 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#C#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:51,743 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/6582bdc609b049ed8da67b2aa49b143e is 50, key is test_row_0/C:col10/1731708950506/Put/seqid=0 2024-11-15T22:15:51,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709011738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:51,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741933_1109 (size=12301) 2024-11-15T22:15:51,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/c418b178568b4f32ad2d4debe8206568 2024-11-15T22:15:51,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741934_1110 (size=13357) 2024-11-15T22:15:51,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/360c5d5b6e044b13abd186bd38fb7837 is 50, key is test_row_0/C:col10/1731708950940/Put/seqid=0 2024-11-15T22:15:51,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:15:51,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:51,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:51,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709011845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:51,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709011845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:51,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:51,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709011861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:51,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:51,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:51,892 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/6582bdc609b049ed8da67b2aa49b143e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/6582bdc609b049ed8da67b2aa49b143e 2024-11-15T22:15:51,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741935_1111 (size=12301) 2024-11-15T22:15:51,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/360c5d5b6e044b13abd186bd38fb7837 2024-11-15T22:15:51,941 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/C of 0cf44e7263c9528c718ed800ce4e85e9 into 6582bdc609b049ed8da67b2aa49b143e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:51,941 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:51,941 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/C, priority=13, startTime=1731708951548; duration=0sec 2024-11-15T22:15:51,941 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:51,942 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:C 2024-11-15T22:15:51,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/26c25acc05d94759995a243d4038d862 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/26c25acc05d94759995a243d4038d862 2024-11-15T22:15:51,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/26c25acc05d94759995a243d4038d862, entries=150, sequenceid=444, filesize=12.0 K 2024-11-15T22:15:51,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/c418b178568b4f32ad2d4debe8206568 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c418b178568b4f32ad2d4debe8206568 2024-11-15T22:15:52,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:15:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:52,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c418b178568b4f32ad2d4debe8206568, entries=150, sequenceid=444, filesize=12.0 K 2024-11-15T22:15:52,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/360c5d5b6e044b13abd186bd38fb7837 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/360c5d5b6e044b13abd186bd38fb7837 2024-11-15T22:15:52,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/360c5d5b6e044b13abd186bd38fb7837, entries=150, sequenceid=444, filesize=12.0 K 2024-11-15T22:15:52,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T22:15:52,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0cf44e7263c9528c718ed800ce4e85e9 in 407ms, sequenceid=444, compaction requested=false 2024-11-15T22:15:52,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:52,068 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/a0975f2144704f86812c3e56e67fb2c2 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/a0975f2144704f86812c3e56e67fb2c2 2024-11-15T22:15:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:52,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T22:15:52,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731708951732, max timestamp=1731708952092Num uniques 859; 2024-11-15T22:15:52,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:15:52,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731708951732, max timestamp=1731708952092Num uniques 859; 2024-11-15T22:15:52,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:15:52,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708952069, max timestamp=1731708952091Num uniques 859; 2024-11-15T22:15:52,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:15:52,121 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/A of 0cf44e7263c9528c718ed800ce4e85e9 into a0975f2144704f86812c3e56e67fb2c2(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:52,121 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:52,121 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/A, priority=13, startTime=1731708951548; duration=0sec 2024-11-15T22:15:52,121 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:52,121 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:A 2024-11-15T22:15:52,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:52,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:15:52,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:52,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:15:52,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:52,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:15:52,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:15:52,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/63d3566794ec40ceb9eaa46bb9ac3c38 is 50, key is test_row_0/A:col10/1731708952080/Put/seqid=0 2024-11-15T22:15:52,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:52,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:52,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:52,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741936_1112 (size=12301) 2024-11-15T22:15:52,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/63d3566794ec40ceb9eaa46bb9ac3c38 2024-11-15T22:15:52,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/324035afc448487fa5a689eaedda2cc4 is 50, key is test_row_0/B:col10/1731708952080/Put/seqid=0 2024-11-15T22:15:52,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741937_1113 (size=12301) 2024-11-15T22:15:52,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:15:52,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:52,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:52,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:52,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731708952285, max timestamp=1731708952339Num uniques 859; 2024-11-15T22:15:52,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T22:15:52,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731708952285, max timestamp=1731708952339Num uniques 859; 2024-11-15T22:15:52,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T22:15:52,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731708952285, max timestamp=1731708952339Num uniques 859; 2024-11-15T22:15:52,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T22:15:52,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:52,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709012416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709012415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:52,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:52,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709012417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:52,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:15:52,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:52,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:52,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:52,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709012531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:52,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:52,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709012532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709012532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:52,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:52,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:15:52,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:52,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:52,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:52,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/324035afc448487fa5a689eaedda2cc4 2024-11-15T22:15:52,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/be8db2ac07e048bd9f9b56e7030a015a is 50, key is test_row_0/C:col10/1731708952080/Put/seqid=0 2024-11-15T22:15:52,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709012742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:52,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709012745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:52,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709012742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:52,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:52,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:52,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:52,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741938_1114 (size=12301) 2024-11-15T22:15:52,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:15:52,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:52,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:52,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:52,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:15:52,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:52,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:52,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:52,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T22:15:53,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709013062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:53,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709013062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:53,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709013072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:53,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:15:53,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:53,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:53,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:53,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:53,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:15:53,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/be8db2ac07e048bd9f9b56e7030a015a 2024-11-15T22:15:53,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/63d3566794ec40ceb9eaa46bb9ac3c38 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/63d3566794ec40ceb9eaa46bb9ac3c38 2024-11-15T22:15:53,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/63d3566794ec40ceb9eaa46bb9ac3c38, entries=150, sequenceid=469, filesize=12.0 K 2024-11-15T22:15:53,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/324035afc448487fa5a689eaedda2cc4 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/324035afc448487fa5a689eaedda2cc4 2024-11-15T22:15:53,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/324035afc448487fa5a689eaedda2cc4, entries=150, sequenceid=469, filesize=12.0 K 2024-11-15T22:15:53,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/be8db2ac07e048bd9f9b56e7030a015a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/be8db2ac07e048bd9f9b56e7030a015a 2024-11-15T22:15:53,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/be8db2ac07e048bd9f9b56e7030a015a, entries=150, sequenceid=469, filesize=12.0 K 2024-11-15T22:15:53,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~38.02 KB/38930, heapSize ~96.27 KB/98576, currentSize=154.31 KB/158010 for 0cf44e7263c9528c718ed800ce4e85e9 in 1129ms, sequenceid=469, compaction requested=true 2024-11-15T22:15:53,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:53,225 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:53,226 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:53,227 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/A is initiating minor compaction (all files) 2024-11-15T22:15:53,227 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/A in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:53,227 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/a0975f2144704f86812c3e56e67fb2c2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/26c25acc05d94759995a243d4038d862, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/63d3566794ec40ceb9eaa46bb9ac3c38] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=37.1 K 2024-11-15T22:15:53,228 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0975f2144704f86812c3e56e67fb2c2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731708950506 2024-11-15T22:15:53,229 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26c25acc05d94759995a243d4038d862, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731708950936 2024-11-15T22:15:53,229 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63d3566794ec40ceb9eaa46bb9ac3c38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731708952080 2024-11-15T22:15:53,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:15:53,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:53,237 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:53,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:15:53,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:53,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:15:53,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:53,239 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:53,239 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/B is initiating minor compaction (all files) 2024-11-15T22:15:53,239 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/B in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:53,239 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/ff549231cfe849babd43e5155d393587, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c418b178568b4f32ad2d4debe8206568, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/324035afc448487fa5a689eaedda2cc4] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=37.1 K 2024-11-15T22:15:53,244 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting ff549231cfe849babd43e5155d393587, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731708950506 2024-11-15T22:15:53,249 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c418b178568b4f32ad2d4debe8206568, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731708950936 2024-11-15T22:15:53,250 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 324035afc448487fa5a689eaedda2cc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731708952080 2024-11-15T22:15:53,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:15:53,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:53,265 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-11-15T22:15:53,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:53,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:15:53,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:53,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:15:53,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:53,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:15:53,280 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#A#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:53,281 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/f6094c4bb2154e79972ef97ee4a90ccc is 50, key is test_row_0/A:col10/1731708952080/Put/seqid=0 2024-11-15T22:15:53,295 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#B#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:53,296 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/b98488694457475ea0df51e5803057d9 is 50, key is test_row_0/B:col10/1731708952080/Put/seqid=0 2024-11-15T22:15:53,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/74c26cd6209c4ce4821e5c4c9b21776a is 50, key is test_row_0/A:col10/1731708952413/Put/seqid=0 2024-11-15T22:15:53,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741939_1115 (size=13459) 2024-11-15T22:15:53,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741941_1117 (size=12301) 2024-11-15T22:15:53,369 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/f6094c4bb2154e79972ef97ee4a90ccc as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f6094c4bb2154e79972ef97ee4a90ccc 2024-11-15T22:15:53,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741940_1116 (size=13459) 2024-11-15T22:15:53,393 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/b98488694457475ea0df51e5803057d9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/b98488694457475ea0df51e5803057d9 2024-11-15T22:15:53,405 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/B of 0cf44e7263c9528c718ed800ce4e85e9 into b98488694457475ea0df51e5803057d9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:53,406 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:53,406 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/B, priority=13, startTime=1731708953237; duration=0sec 2024-11-15T22:15:53,406 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/A of 0cf44e7263c9528c718ed800ce4e85e9 into f6094c4bb2154e79972ef97ee4a90ccc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:53,406 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:53,406 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/A, priority=13, startTime=1731708953225; duration=0sec 2024-11-15T22:15:53,406 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:53,406 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:A 2024-11-15T22:15:53,406 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:53,406 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:53,406 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:B 2024-11-15T22:15:53,409 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:53,409 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/C is initiating minor compaction (all files) 2024-11-15T22:15:53,409 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/C in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:53,409 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/6582bdc609b049ed8da67b2aa49b143e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/360c5d5b6e044b13abd186bd38fb7837, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/be8db2ac07e048bd9f9b56e7030a015a] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=37.1 K 2024-11-15T22:15:53,410 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6582bdc609b049ed8da67b2aa49b143e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731708950506 2024-11-15T22:15:53,410 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 360c5d5b6e044b13abd186bd38fb7837, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731708950936 2024-11-15T22:15:53,412 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting be8db2ac07e048bd9f9b56e7030a015a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731708952080 2024-11-15T22:15:53,430 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#C#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:53,431 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/94099912289e41efb49d81aa2eefef05 is 50, key is test_row_0/C:col10/1731708952080/Put/seqid=0 2024-11-15T22:15:53,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741942_1118 (size=13459) 2024-11-15T22:15:53,481 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/94099912289e41efb49d81aa2eefef05 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/94099912289e41efb49d81aa2eefef05 2024-11-15T22:15:53,507 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/C of 0cf44e7263c9528c718ed800ce4e85e9 into 94099912289e41efb49d81aa2eefef05(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:53,507 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:53,507 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/C, priority=13, startTime=1731708953238; duration=0sec 2024-11-15T22:15:53,507 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:53,507 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:C 2024-11-15T22:15:53,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:53,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. as already flushing 2024-11-15T22:15:53,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38262 deadline: 1731709013591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:53,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709013596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:53,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:53,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709013596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:53,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:53,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709013712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:53,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709013712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:53,758 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/74c26cd6209c4ce4821e5c4c9b21776a 2024-11-15T22:15:53,765 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-15T22:15:53,765 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:15:53,766 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-15T22:15:53,766 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:15:53,770 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-15T22:15:53,770 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:15:53,770 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-15T22:15:53,770 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:15:53,772 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:53,772 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:53,772 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:53,772 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:53,772 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:53,772 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:15:53,772 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:15:53,773 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:53,773 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:53,773 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:15:53,775 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:53,775 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:15:53,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/2494c7c500b94425832681209ddcdfe9 is 50, key is test_row_0/B:col10/1731708952413/Put/seqid=0 2024-11-15T22:15:53,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741943_1119 (size=12301) 2024-11-15T22:15:53,782 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/2494c7c500b94425832681209ddcdfe9 2024-11-15T22:15:53,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/64bb42c47a5b489b8135a424cc48336a is 50, key is test_row_0/C:col10/1731708952413/Put/seqid=0 2024-11-15T22:15:53,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741944_1120 (size=12301) 2024-11-15T22:15:53,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709013924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:53,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709013932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:53,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:53,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:54,223 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/64bb42c47a5b489b8135a424cc48336a 2024-11-15T22:15:54,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/74c26cd6209c4ce4821e5c4c9b21776a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/74c26cd6209c4ce4821e5c4c9b21776a 2024-11-15T22:15:54,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38280 deadline: 1731709014231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:54,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:54,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:54,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:54,235 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/74c26cd6209c4ce4821e5c4c9b21776a, entries=150, sequenceid=505, filesize=12.0 K 2024-11-15T22:15:54,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/2494c7c500b94425832681209ddcdfe9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/2494c7c500b94425832681209ddcdfe9 2024-11-15T22:15:54,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:15:54,242 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/2494c7c500b94425832681209ddcdfe9, entries=150, sequenceid=505, filesize=12.0 K 2024-11-15T22:15:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38256 deadline: 1731709014241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 2024-11-15T22:15:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:15:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:15:54,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/64bb42c47a5b489b8135a424cc48336a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/64bb42c47a5b489b8135a424cc48336a 2024-11-15T22:15:54,250 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/64bb42c47a5b489b8135a424cc48336a, entries=150, sequenceid=505, filesize=12.0 K 2024-11-15T22:15:54,251 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=33.54 KB/34350 for 0cf44e7263c9528c718ed800ce4e85e9 in 986ms, sequenceid=505, compaction requested=false 2024-11-15T22:15:54,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:54,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:54,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-15T22:15:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-15T22:15:54,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-15T22:15:54,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3740 sec 2024-11-15T22:15:54,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 3.3830 sec 2024-11-15T22:15:54,613 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-15T22:15:54,613 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:15:54,613 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:54,613 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:54,613 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:15:54,649 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T22:15:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:54,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T22:15:54,744 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-15T22:15:54,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:54,744 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:15:54,744 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:54,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:54,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:54,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:54,745 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:54,745 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:15:54,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:54,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:54,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/3d34b5da0ea8415685c0f5cac5f2c840 is 50, key is test_row_0/A:col10/1731708954611/Put/seqid=0 2024-11-15T22:15:54,754 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-15T22:15:54,754 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:15:54,754 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:54,754 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:54,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:15:54,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741945_1121 (size=9857) 2024-11-15T22:15:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T22:15:55,062 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:15:55,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/3d34b5da0ea8415685c0f5cac5f2c840 2024-11-15T22:15:55,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/10b0366df90d4294b8add60b214d1641 is 50, key is test_row_0/B:col10/1731708954611/Put/seqid=0 2024-11-15T22:15:55,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741946_1122 (size=9857) 2024-11-15T22:15:55,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/10b0366df90d4294b8add60b214d1641 2024-11-15T22:15:55,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/eea517cc7bb0457eb64519dd8b3c4c45 is 50, key is test_row_0/C:col10/1731708954611/Put/seqid=0 2024-11-15T22:15:55,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741947_1123 (size=9857) 2024-11-15T22:15:56,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/eea517cc7bb0457eb64519dd8b3c4c45 2024-11-15T22:15:56,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/3d34b5da0ea8415685c0f5cac5f2c840 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/3d34b5da0ea8415685c0f5cac5f2c840 2024-11-15T22:15:56,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/3d34b5da0ea8415685c0f5cac5f2c840, entries=100, sequenceid=518, filesize=9.6 K 2024-11-15T22:15:56,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/10b0366df90d4294b8add60b214d1641 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/10b0366df90d4294b8add60b214d1641 2024-11-15T22:15:56,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/10b0366df90d4294b8add60b214d1641, entries=100, sequenceid=518, filesize=9.6 K 2024-11-15T22:15:56,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/eea517cc7bb0457eb64519dd8b3c4c45 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/eea517cc7bb0457eb64519dd8b3c4c45 2024-11-15T22:15:56,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/eea517cc7bb0457eb64519dd8b3c4c45, entries=100, sequenceid=518, filesize=9.6 K 2024-11-15T22:15:56,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=6.71 KB/6870 for 0cf44e7263c9528c718ed800ce4e85e9 in 1320ms, sequenceid=518, compaction requested=true 2024-11-15T22:15:56,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:56,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:15:56,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:56,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:15:56,065 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:56,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:56,065 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:56,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0cf44e7263c9528c718ed800ce4e85e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:15:56,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:56,066 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:56,066 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:56,066 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/A is initiating minor compaction (all files) 2024-11-15T22:15:56,066 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/B is initiating minor compaction (all files) 2024-11-15T22:15:56,066 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/A in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:56,066 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/B in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:56,066 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f6094c4bb2154e79972ef97ee4a90ccc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/74c26cd6209c4ce4821e5c4c9b21776a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/3d34b5da0ea8415685c0f5cac5f2c840] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=34.8 K 2024-11-15T22:15:56,066 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/b98488694457475ea0df51e5803057d9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/2494c7c500b94425832681209ddcdfe9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/10b0366df90d4294b8add60b214d1641] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=34.8 K 2024-11-15T22:15:56,067 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6094c4bb2154e79972ef97ee4a90ccc, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731708952080 2024-11-15T22:15:56,067 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting b98488694457475ea0df51e5803057d9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731708952080 2024-11-15T22:15:56,067 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74c26cd6209c4ce4821e5c4c9b21776a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1731708952383 2024-11-15T22:15:56,067 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 2494c7c500b94425832681209ddcdfe9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1731708952383 2024-11-15T22:15:56,067 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d34b5da0ea8415685c0f5cac5f2c840, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731708954611 2024-11-15T22:15:56,067 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 10b0366df90d4294b8add60b214d1641, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731708954611 2024-11-15T22:15:56,075 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:56,075 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/0845a0415ad443349fdf6634798b7a19 is 50, key is test_row_0/B:col10/1731708954611/Put/seqid=0 2024-11-15T22:15:56,078 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#A#compaction#112 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:56,079 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/0448bd458e25475e9e111c62d240cbbb is 50, key is test_row_0/A:col10/1731708954611/Put/seqid=0 2024-11-15T22:15:56,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741948_1124 (size=13561) 2024-11-15T22:15:56,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741949_1125 (size=13561) 2024-11-15T22:15:56,485 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/0845a0415ad443349fdf6634798b7a19 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/0845a0415ad443349fdf6634798b7a19 2024-11-15T22:15:56,487 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/0448bd458e25475e9e111c62d240cbbb as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/0448bd458e25475e9e111c62d240cbbb 2024-11-15T22:15:56,494 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/B of 0cf44e7263c9528c718ed800ce4e85e9 into 0845a0415ad443349fdf6634798b7a19(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:56,494 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/A of 0cf44e7263c9528c718ed800ce4e85e9 into 0448bd458e25475e9e111c62d240cbbb(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:56,495 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:56,495 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:56,495 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/B, priority=13, startTime=1731708956065; duration=0sec 2024-11-15T22:15:56,495 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/A, priority=13, startTime=1731708956065; duration=0sec 2024-11-15T22:15:56,495 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:15:56,495 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:A 2024-11-15T22:15:56,495 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:56,495 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:B 2024-11-15T22:15:56,495 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:15:56,496 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:15:56,496 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 0cf44e7263c9528c718ed800ce4e85e9/C is initiating minor compaction (all files) 2024-11-15T22:15:56,496 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0cf44e7263c9528c718ed800ce4e85e9/C in TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:56,497 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/94099912289e41efb49d81aa2eefef05, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/64bb42c47a5b489b8135a424cc48336a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/eea517cc7bb0457eb64519dd8b3c4c45] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp, totalSize=34.8 K 2024-11-15T22:15:56,497 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94099912289e41efb49d81aa2eefef05, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731708952080 2024-11-15T22:15:56,497 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64bb42c47a5b489b8135a424cc48336a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1731708952383 2024-11-15T22:15:56,497 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting eea517cc7bb0457eb64519dd8b3c4c45, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731708954611 2024-11-15T22:15:56,506 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0cf44e7263c9528c718ed800ce4e85e9#C#compaction#113 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:15:56,507 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/48f9ac9829224bd8b00c89ed646fa6c7 is 50, key is test_row_0/C:col10/1731708954611/Put/seqid=0 2024-11-15T22:15:56,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741950_1126 (size=13561) 2024-11-15T22:15:56,888 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-15T22:15:56,888 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:15:56,888 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:56,888 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:56,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:15:56,927 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/48f9ac9829224bd8b00c89ed646fa6c7 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/48f9ac9829224bd8b00c89ed646fa6c7 2024-11-15T22:15:56,935 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0cf44e7263c9528c718ed800ce4e85e9/C of 0cf44e7263c9528c718ed800ce4e85e9 into 48f9ac9829224bd8b00c89ed646fa6c7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:15:56,936 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0cf44e7263c9528c718ed800ce4e85e9: 2024-11-15T22:15:56,936 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., storeName=0cf44e7263c9528c718ed800ce4e85e9/C, priority=13, startTime=1731708956065; duration=0sec 2024-11-15T22:15:56,936 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:15:56,936 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0cf44e7263c9528c718ed800ce4e85e9:C 2024-11-15T22:15:56,946 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-15T22:15:56,946 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:15:56,946 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:56,946 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:56,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:15:56,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T22:15:56,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-15T22:15:56,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-15T22:15:56,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-15T22:15:56,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-15T22:15:56,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 125 2024-11-15T22:15:56,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T22:15:56,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2419 2024-11-15T22:15:56,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2432 2024-11-15T22:15:56,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T22:15:56,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1100 2024-11-15T22:15:56,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3299 rows 2024-11-15T22:15:56,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1114 2024-11-15T22:15:56,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3338 rows 2024-11-15T22:15:56,949 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:15:56,949 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:15:56,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:56,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:15:56,950 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:15:56,957 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T22:15:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T22:15:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T22:15:57,012 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708957011"}]},"ts":"1731708957011"} 2024-11-15T22:15:57,015 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T22:15:57,016 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T22:15:57,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:15:57,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0cf44e7263c9528c718ed800ce4e85e9, UNASSIGN}] 2024-11-15T22:15:57,027 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0cf44e7263c9528c718ed800ce4e85e9, UNASSIGN 2024-11-15T22:15:57,030 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=0cf44e7263c9528c718ed800ce4e85e9, regionState=CLOSING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:15:57,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0cf44e7263c9528c718ed800ce4e85e9, UNASSIGN because future has completed 2024-11-15T22:15:57,040 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:15:57,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:15:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T22:15:57,201 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:57,201 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:15:57,202 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 0cf44e7263c9528c718ed800ce4e85e9, disabling compactions & flushes 2024-11-15T22:15:57,202 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:57,202 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:57,202 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. after waiting 0 ms 2024-11-15T22:15:57,202 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:57,202 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 0cf44e7263c9528c718ed800ce4e85e9 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-15T22:15:57,203 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=A 2024-11-15T22:15:57,203 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:57,203 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=B 2024-11-15T22:15:57,203 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:57,203 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0cf44e7263c9528c718ed800ce4e85e9, store=C 2024-11-15T22:15:57,203 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:15:57,210 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/e8c6ed9905634760b02b11d5e6e6f947 is 50, key is test_row_1/A:col10/1731708956944/Put/seqid=0 2024-11-15T22:15:57,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741951_1127 (size=9857) 2024-11-15T22:15:57,226 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/e8c6ed9905634760b02b11d5e6e6f947 2024-11-15T22:15:57,239 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/465a47bb1e654ab2b3a1a988196d96ba is 50, key is test_row_1/B:col10/1731708956944/Put/seqid=0 2024-11-15T22:15:57,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741952_1128 (size=9857) 2024-11-15T22:15:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T22:15:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T22:15:57,649 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/465a47bb1e654ab2b3a1a988196d96ba 2024-11-15T22:15:57,664 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/ac2fa447df3a49e988f7a166bf1cfd4b is 50, key is test_row_1/C:col10/1731708956944/Put/seqid=0 2024-11-15T22:15:57,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741953_1129 (size=9857) 2024-11-15T22:15:58,069 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/ac2fa447df3a49e988f7a166bf1cfd4b 2024-11-15T22:15:58,075 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/A/e8c6ed9905634760b02b11d5e6e6f947 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/e8c6ed9905634760b02b11d5e6e6f947 2024-11-15T22:15:58,081 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/e8c6ed9905634760b02b11d5e6e6f947, entries=100, sequenceid=527, filesize=9.6 K 2024-11-15T22:15:58,082 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/B/465a47bb1e654ab2b3a1a988196d96ba as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/465a47bb1e654ab2b3a1a988196d96ba 2024-11-15T22:15:58,095 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/465a47bb1e654ab2b3a1a988196d96ba, entries=100, sequenceid=527, filesize=9.6 K 2024-11-15T22:15:58,097 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/.tmp/C/ac2fa447df3a49e988f7a166bf1cfd4b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ac2fa447df3a49e988f7a166bf1cfd4b 2024-11-15T22:15:58,102 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ac2fa447df3a49e988f7a166bf1cfd4b, entries=100, sequenceid=527, filesize=9.6 K 2024-11-15T22:15:58,103 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 0cf44e7263c9528c718ed800ce4e85e9 in 901ms, sequenceid=527, compaction requested=false 2024-11-15T22:15:58,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2b535aeef3fa44c5999f61dff4b42e4c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/a6413c0082af4b0ca2ac852b94da1fd2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7aa245bb4a3c4e989a9868d7c9f463a7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2872caa7bfa24880979cf9aef9689257, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/184691cead8244a1a473957c19f00ef1, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/0e09eab9ffe34e1584a728936830f6b8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/05494c0eeb1043e29858dee9e5eba4fb, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/afb416507734432abbc50bac19426742, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/69369e3437f94691acc18813ff5a68df, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/fc2c5f6f7b074ab2b8a12a63ee517714, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/fd5f84a3733543ada402d2ad6237b4e8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/454e4bf242124075af6932d36a25d7fa, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/24e51f5feac6412ea6c16b37cca0ae0d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7f5e9e461df84566b6a891aaea8d6f03, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/12a6c4989ff941f5a19c1ea857e4a087, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2ff12a31af0749129dc5f7e14e1d144a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/73a6269eb675468ea8baeadd059411aa, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f0c76803852647539b3e79f407f59f09, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f8c0df7acbb24349a8b81b4a624be476, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/0ed8ef10e73a4c5bbe632fa93a423c88, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/4c4cd510d6ce420a9b6f2b8084898787, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/d130f2c08b9e4a08992ce4fee866d24c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/6bf11aece7154541b89e821c123791fc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/72429b7d77a64553b89a3070dfcd0d93, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/4b9515d179a44795bef081a7c96bd59e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/43b8d8e66d784ef8b31d0325a53ac21d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/913f5e0cf2a94d26b131e2af89bd4669, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/5d5194cbafe24364972462ba763e4a62, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/1f361bf5612e40ecb7ef243d60b0cbf6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/62e26eae80db4953a116e8f98ce33ee2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7ee8f8ef787b4b5ba297a118a2484be8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/a0975f2144704f86812c3e56e67fb2c2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/26c25acc05d94759995a243d4038d862, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f6094c4bb2154e79972ef97ee4a90ccc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/63d3566794ec40ceb9eaa46bb9ac3c38, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/74c26cd6209c4ce4821e5c4c9b21776a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/3d34b5da0ea8415685c0f5cac5f2c840] to archive 2024-11-15T22:15:58,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:15:58,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T22:15:58,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2b535aeef3fa44c5999f61dff4b42e4c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2b535aeef3fa44c5999f61dff4b42e4c 2024-11-15T22:15:58,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/a6413c0082af4b0ca2ac852b94da1fd2 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/a6413c0082af4b0ca2ac852b94da1fd2 2024-11-15T22:15:58,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7aa245bb4a3c4e989a9868d7c9f463a7 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7aa245bb4a3c4e989a9868d7c9f463a7 2024-11-15T22:15:58,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2872caa7bfa24880979cf9aef9689257 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2872caa7bfa24880979cf9aef9689257 2024-11-15T22:15:58,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/184691cead8244a1a473957c19f00ef1 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/184691cead8244a1a473957c19f00ef1 2024-11-15T22:15:58,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/0e09eab9ffe34e1584a728936830f6b8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/0e09eab9ffe34e1584a728936830f6b8 2024-11-15T22:15:58,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/05494c0eeb1043e29858dee9e5eba4fb to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/05494c0eeb1043e29858dee9e5eba4fb 2024-11-15T22:15:58,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/afb416507734432abbc50bac19426742 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/afb416507734432abbc50bac19426742 2024-11-15T22:15:58,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/69369e3437f94691acc18813ff5a68df to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/69369e3437f94691acc18813ff5a68df 2024-11-15T22:15:58,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/fc2c5f6f7b074ab2b8a12a63ee517714 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/fc2c5f6f7b074ab2b8a12a63ee517714 2024-11-15T22:15:58,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/fd5f84a3733543ada402d2ad6237b4e8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/fd5f84a3733543ada402d2ad6237b4e8 2024-11-15T22:15:58,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/454e4bf242124075af6932d36a25d7fa to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/454e4bf242124075af6932d36a25d7fa 2024-11-15T22:15:58,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/24e51f5feac6412ea6c16b37cca0ae0d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/24e51f5feac6412ea6c16b37cca0ae0d 2024-11-15T22:15:58,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7f5e9e461df84566b6a891aaea8d6f03 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7f5e9e461df84566b6a891aaea8d6f03 2024-11-15T22:15:58,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/12a6c4989ff941f5a19c1ea857e4a087 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/12a6c4989ff941f5a19c1ea857e4a087 2024-11-15T22:15:58,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2ff12a31af0749129dc5f7e14e1d144a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/2ff12a31af0749129dc5f7e14e1d144a 2024-11-15T22:15:58,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/73a6269eb675468ea8baeadd059411aa to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/73a6269eb675468ea8baeadd059411aa 2024-11-15T22:15:58,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f0c76803852647539b3e79f407f59f09 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f0c76803852647539b3e79f407f59f09 2024-11-15T22:15:58,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f8c0df7acbb24349a8b81b4a624be476 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f8c0df7acbb24349a8b81b4a624be476 2024-11-15T22:15:58,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/0ed8ef10e73a4c5bbe632fa93a423c88 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/0ed8ef10e73a4c5bbe632fa93a423c88 2024-11-15T22:15:58,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/4c4cd510d6ce420a9b6f2b8084898787 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/4c4cd510d6ce420a9b6f2b8084898787 2024-11-15T22:15:58,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/d130f2c08b9e4a08992ce4fee866d24c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/d130f2c08b9e4a08992ce4fee866d24c 2024-11-15T22:15:58,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/6bf11aece7154541b89e821c123791fc to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/6bf11aece7154541b89e821c123791fc 2024-11-15T22:15:58,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/72429b7d77a64553b89a3070dfcd0d93 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/72429b7d77a64553b89a3070dfcd0d93 2024-11-15T22:15:58,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/4b9515d179a44795bef081a7c96bd59e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/4b9515d179a44795bef081a7c96bd59e 2024-11-15T22:15:58,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/43b8d8e66d784ef8b31d0325a53ac21d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/43b8d8e66d784ef8b31d0325a53ac21d 2024-11-15T22:15:58,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/913f5e0cf2a94d26b131e2af89bd4669 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/913f5e0cf2a94d26b131e2af89bd4669 2024-11-15T22:15:58,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/5d5194cbafe24364972462ba763e4a62 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/5d5194cbafe24364972462ba763e4a62 2024-11-15T22:15:58,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/1f361bf5612e40ecb7ef243d60b0cbf6 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/1f361bf5612e40ecb7ef243d60b0cbf6 2024-11-15T22:15:58,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/62e26eae80db4953a116e8f98ce33ee2 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/62e26eae80db4953a116e8f98ce33ee2 2024-11-15T22:15:58,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7ee8f8ef787b4b5ba297a118a2484be8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/7ee8f8ef787b4b5ba297a118a2484be8 2024-11-15T22:15:58,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/a0975f2144704f86812c3e56e67fb2c2 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/a0975f2144704f86812c3e56e67fb2c2 2024-11-15T22:15:58,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/26c25acc05d94759995a243d4038d862 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/26c25acc05d94759995a243d4038d862 2024-11-15T22:15:58,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f6094c4bb2154e79972ef97ee4a90ccc to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/f6094c4bb2154e79972ef97ee4a90ccc 2024-11-15T22:15:58,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/63d3566794ec40ceb9eaa46bb9ac3c38 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/63d3566794ec40ceb9eaa46bb9ac3c38 2024-11-15T22:15:58,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/74c26cd6209c4ce4821e5c4c9b21776a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/74c26cd6209c4ce4821e5c4c9b21776a 2024-11-15T22:15:58,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/3d34b5da0ea8415685c0f5cac5f2c840 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/3d34b5da0ea8415685c0f5cac5f2c840 2024-11-15T22:15:58,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c72061c580bc40b4a106ebf8a3f240eb, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/1af2b849e8554562a706c818e522fe3e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/950455225d2340588ae5dfa972d095e2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/23a4272a2b9d41f8a2918da955cfa499, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/4d882ef8a9cd4881808bdfe85220ddc3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c025da8e7c674fac94728da69b7b106e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d4a4740bc1bf4b83b654a893886bc6fe, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/f7eb96a796134aa0bb18d278ddd7b1f9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/b78c3b250c5c40019bd90f5f1107f754, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c0833280e3a141d8b0ce9118c8e97887, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/af36ff097af84538abf2448f4ce1aba6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/2bd96a1528ec474181fa192ff09035ab, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/204a062f4dc3469cb3a32f6d5cccd8c5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c337a1410c4e4df6b32c785d4cd3d823, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d5b8c99767ba44df8c405f02951f0d2e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/758bd910162140dbb6029e16eabfe585, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/6a22f8993d734064b5019c91106ebac4, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/33feb232565d47d18cb2d55226e8028d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/af9f5d56fefb4992bec621232bfe67a5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d80c62a3b00444bc85e96be1c08f69de, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/4e5a4612dd1c4ccaadde83d11358f748, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/0306fc712ae242c68fce5f7ba38e88a3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/6eb79b34508a4e3ba7cf34b9c05d3f33, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/3a8e078fec6f4f1d86684770569f81d3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/ab0a275e2ad546c0a778b319e8af4ec3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/99bfbd92cca84d768f5f352db8174a10, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/edbb2ebaed024d54bafeb9ef4213b6b3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/aae6f4a43f3249379f3bff06c16ae9c2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/5c03e95ea2454ba1b0e444d36f3d9985, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/083b06be052547e2903c162537e1078c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/ff549231cfe849babd43e5155d393587, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/9e0b21788a1047ea96d87de30563f6ac, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c418b178568b4f32ad2d4debe8206568, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/b98488694457475ea0df51e5803057d9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/324035afc448487fa5a689eaedda2cc4, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/2494c7c500b94425832681209ddcdfe9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/10b0366df90d4294b8add60b214d1641] to archive 2024-11-15T22:15:58,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:15:58,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c72061c580bc40b4a106ebf8a3f240eb to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c72061c580bc40b4a106ebf8a3f240eb 2024-11-15T22:15:58,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/1af2b849e8554562a706c818e522fe3e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/1af2b849e8554562a706c818e522fe3e 2024-11-15T22:15:58,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/950455225d2340588ae5dfa972d095e2 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/950455225d2340588ae5dfa972d095e2 2024-11-15T22:15:58,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/23a4272a2b9d41f8a2918da955cfa499 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/23a4272a2b9d41f8a2918da955cfa499 2024-11-15T22:15:58,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/4d882ef8a9cd4881808bdfe85220ddc3 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/4d882ef8a9cd4881808bdfe85220ddc3 2024-11-15T22:15:58,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c025da8e7c674fac94728da69b7b106e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c025da8e7c674fac94728da69b7b106e 2024-11-15T22:15:58,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d4a4740bc1bf4b83b654a893886bc6fe to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d4a4740bc1bf4b83b654a893886bc6fe 2024-11-15T22:15:58,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/f7eb96a796134aa0bb18d278ddd7b1f9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/f7eb96a796134aa0bb18d278ddd7b1f9 2024-11-15T22:15:58,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/b78c3b250c5c40019bd90f5f1107f754 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/b78c3b250c5c40019bd90f5f1107f754 2024-11-15T22:15:58,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c0833280e3a141d8b0ce9118c8e97887 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c0833280e3a141d8b0ce9118c8e97887 2024-11-15T22:15:58,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/af36ff097af84538abf2448f4ce1aba6 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/af36ff097af84538abf2448f4ce1aba6 2024-11-15T22:15:58,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/2bd96a1528ec474181fa192ff09035ab to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/2bd96a1528ec474181fa192ff09035ab 2024-11-15T22:15:58,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/204a062f4dc3469cb3a32f6d5cccd8c5 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/204a062f4dc3469cb3a32f6d5cccd8c5 2024-11-15T22:15:58,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c337a1410c4e4df6b32c785d4cd3d823 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c337a1410c4e4df6b32c785d4cd3d823 2024-11-15T22:15:58,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d5b8c99767ba44df8c405f02951f0d2e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d5b8c99767ba44df8c405f02951f0d2e 2024-11-15T22:15:58,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/758bd910162140dbb6029e16eabfe585 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/758bd910162140dbb6029e16eabfe585 2024-11-15T22:15:58,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/6a22f8993d734064b5019c91106ebac4 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/6a22f8993d734064b5019c91106ebac4 2024-11-15T22:15:58,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/33feb232565d47d18cb2d55226e8028d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/33feb232565d47d18cb2d55226e8028d 2024-11-15T22:15:58,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/af9f5d56fefb4992bec621232bfe67a5 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/af9f5d56fefb4992bec621232bfe67a5 2024-11-15T22:15:58,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d80c62a3b00444bc85e96be1c08f69de to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/d80c62a3b00444bc85e96be1c08f69de 2024-11-15T22:15:58,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/4e5a4612dd1c4ccaadde83d11358f748 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/4e5a4612dd1c4ccaadde83d11358f748 2024-11-15T22:15:58,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/0306fc712ae242c68fce5f7ba38e88a3 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/0306fc712ae242c68fce5f7ba38e88a3 2024-11-15T22:15:58,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/6eb79b34508a4e3ba7cf34b9c05d3f33 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/6eb79b34508a4e3ba7cf34b9c05d3f33 2024-11-15T22:15:58,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/3a8e078fec6f4f1d86684770569f81d3 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/3a8e078fec6f4f1d86684770569f81d3 2024-11-15T22:15:58,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/ab0a275e2ad546c0a778b319e8af4ec3 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/ab0a275e2ad546c0a778b319e8af4ec3 2024-11-15T22:15:58,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/99bfbd92cca84d768f5f352db8174a10 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/99bfbd92cca84d768f5f352db8174a10 2024-11-15T22:15:58,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/edbb2ebaed024d54bafeb9ef4213b6b3 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/edbb2ebaed024d54bafeb9ef4213b6b3 2024-11-15T22:15:58,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/aae6f4a43f3249379f3bff06c16ae9c2 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/aae6f4a43f3249379f3bff06c16ae9c2 2024-11-15T22:15:58,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/5c03e95ea2454ba1b0e444d36f3d9985 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/5c03e95ea2454ba1b0e444d36f3d9985 2024-11-15T22:15:58,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/083b06be052547e2903c162537e1078c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/083b06be052547e2903c162537e1078c 2024-11-15T22:15:58,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/ff549231cfe849babd43e5155d393587 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/ff549231cfe849babd43e5155d393587 2024-11-15T22:15:58,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/9e0b21788a1047ea96d87de30563f6ac to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/9e0b21788a1047ea96d87de30563f6ac 2024-11-15T22:15:58,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c418b178568b4f32ad2d4debe8206568 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/c418b178568b4f32ad2d4debe8206568 2024-11-15T22:15:58,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/b98488694457475ea0df51e5803057d9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/b98488694457475ea0df51e5803057d9 2024-11-15T22:15:58,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/324035afc448487fa5a689eaedda2cc4 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/324035afc448487fa5a689eaedda2cc4 2024-11-15T22:15:58,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/2494c7c500b94425832681209ddcdfe9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/2494c7c500b94425832681209ddcdfe9 2024-11-15T22:15:58,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/10b0366df90d4294b8add60b214d1641 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/10b0366df90d4294b8add60b214d1641 2024-11-15T22:15:58,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/cda921b7380d4f50830c7f033019deb8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ea26ed682ae042308c336c9f482e0c53, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ed5dfcaab8f8440981b49e9e70c0f50a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/f7102f89913a4e63b9100af3da21a47c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ec72505521fe4eef8415ab989981f95c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/9d74eb80b2c447a08f0fc1dee997eca6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/11c2f79abed04086b25d4d1d89ea8cc1, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8081e8abe30845cbaff05faf487da967, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/682f11a96b424676a70068b6c7358e20, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/0349182a140442be84a67c02046c6277, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/408698720aa94510a52d6f489748c1da, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7a7b81ae30a542c0ac3118b8f3fa6003, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/426ad5360ce44327bd5e132373817bae, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7ac80ecedad04769aac3a95ebe2684cc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/908d88dba55a4e36828e081c1126961f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/dadb0f72baad4500ab1736f3d7987f68, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/4894ee26390144ca9c79f05f49294822, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/c2eab01d88f34b9f93b32302caccf267, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a49048b6877646cbb71b3d58c4833f99, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a198b6d34545475b93ce63c93ff7c0bf, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/62fa340ff3b349a88f573672fdb99b82, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a8e0a27d87a846ecb642f79e59006d9e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8fa9626afc2a4656998f8f0603266fed, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7362289961fb4d1789f3bd00e58c58a7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7f8d5058ec2e4a719d819a84b177f02d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/fd2e0c8683ba45ab9ca92350eec54147, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/91a93855915b459fb3c5c09d433c0d00, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/d27d1ecaee6f4719a3ee0b4e5fb54dcf, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/f01a600344fb48179214c838c8654163, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/d51a89da0ea64d10aaf827143d3616c0, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/6582bdc609b049ed8da67b2aa49b143e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8c17c6dbdb424e3281e2acb1b52ee4d7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/360c5d5b6e044b13abd186bd38fb7837, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/94099912289e41efb49d81aa2eefef05, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/be8db2ac07e048bd9f9b56e7030a015a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/64bb42c47a5b489b8135a424cc48336a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/eea517cc7bb0457eb64519dd8b3c4c45] to archive 2024-11-15T22:15:58,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:15:58,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/cda921b7380d4f50830c7f033019deb8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/cda921b7380d4f50830c7f033019deb8 2024-11-15T22:15:58,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ea26ed682ae042308c336c9f482e0c53 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ea26ed682ae042308c336c9f482e0c53 2024-11-15T22:15:58,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ed5dfcaab8f8440981b49e9e70c0f50a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ed5dfcaab8f8440981b49e9e70c0f50a 2024-11-15T22:15:58,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/f7102f89913a4e63b9100af3da21a47c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/f7102f89913a4e63b9100af3da21a47c 2024-11-15T22:15:58,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ec72505521fe4eef8415ab989981f95c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ec72505521fe4eef8415ab989981f95c 2024-11-15T22:15:58,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/9d74eb80b2c447a08f0fc1dee997eca6 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/9d74eb80b2c447a08f0fc1dee997eca6 2024-11-15T22:15:58,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/11c2f79abed04086b25d4d1d89ea8cc1 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/11c2f79abed04086b25d4d1d89ea8cc1 2024-11-15T22:15:58,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8081e8abe30845cbaff05faf487da967 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8081e8abe30845cbaff05faf487da967 2024-11-15T22:15:58,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/682f11a96b424676a70068b6c7358e20 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/682f11a96b424676a70068b6c7358e20 2024-11-15T22:15:58,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/0349182a140442be84a67c02046c6277 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/0349182a140442be84a67c02046c6277 2024-11-15T22:15:58,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/408698720aa94510a52d6f489748c1da to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/408698720aa94510a52d6f489748c1da 2024-11-15T22:15:58,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7a7b81ae30a542c0ac3118b8f3fa6003 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7a7b81ae30a542c0ac3118b8f3fa6003 2024-11-15T22:15:58,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/426ad5360ce44327bd5e132373817bae to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/426ad5360ce44327bd5e132373817bae 2024-11-15T22:15:58,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7ac80ecedad04769aac3a95ebe2684cc to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7ac80ecedad04769aac3a95ebe2684cc 2024-11-15T22:15:58,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/908d88dba55a4e36828e081c1126961f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/908d88dba55a4e36828e081c1126961f 2024-11-15T22:15:58,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/dadb0f72baad4500ab1736f3d7987f68 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/dadb0f72baad4500ab1736f3d7987f68 2024-11-15T22:15:58,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/4894ee26390144ca9c79f05f49294822 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/4894ee26390144ca9c79f05f49294822 2024-11-15T22:15:58,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/c2eab01d88f34b9f93b32302caccf267 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/c2eab01d88f34b9f93b32302caccf267 2024-11-15T22:15:58,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a49048b6877646cbb71b3d58c4833f99 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a49048b6877646cbb71b3d58c4833f99 2024-11-15T22:15:58,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a198b6d34545475b93ce63c93ff7c0bf to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a198b6d34545475b93ce63c93ff7c0bf 2024-11-15T22:15:58,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/62fa340ff3b349a88f573672fdb99b82 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/62fa340ff3b349a88f573672fdb99b82 2024-11-15T22:15:58,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a8e0a27d87a846ecb642f79e59006d9e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/a8e0a27d87a846ecb642f79e59006d9e 2024-11-15T22:15:58,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8fa9626afc2a4656998f8f0603266fed to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8fa9626afc2a4656998f8f0603266fed 2024-11-15T22:15:58,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7362289961fb4d1789f3bd00e58c58a7 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7362289961fb4d1789f3bd00e58c58a7 2024-11-15T22:15:58,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7f8d5058ec2e4a719d819a84b177f02d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/7f8d5058ec2e4a719d819a84b177f02d 2024-11-15T22:15:58,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/fd2e0c8683ba45ab9ca92350eec54147 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/fd2e0c8683ba45ab9ca92350eec54147 2024-11-15T22:15:58,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/91a93855915b459fb3c5c09d433c0d00 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/91a93855915b459fb3c5c09d433c0d00 2024-11-15T22:15:58,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/d27d1ecaee6f4719a3ee0b4e5fb54dcf to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/d27d1ecaee6f4719a3ee0b4e5fb54dcf 2024-11-15T22:15:58,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/f01a600344fb48179214c838c8654163 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/f01a600344fb48179214c838c8654163 2024-11-15T22:15:58,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/d51a89da0ea64d10aaf827143d3616c0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/d51a89da0ea64d10aaf827143d3616c0 2024-11-15T22:15:58,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/6582bdc609b049ed8da67b2aa49b143e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/6582bdc609b049ed8da67b2aa49b143e 2024-11-15T22:15:58,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8c17c6dbdb424e3281e2acb1b52ee4d7 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/8c17c6dbdb424e3281e2acb1b52ee4d7 2024-11-15T22:15:58,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/360c5d5b6e044b13abd186bd38fb7837 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/360c5d5b6e044b13abd186bd38fb7837 2024-11-15T22:15:58,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/94099912289e41efb49d81aa2eefef05 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/94099912289e41efb49d81aa2eefef05 2024-11-15T22:15:58,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/be8db2ac07e048bd9f9b56e7030a015a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/be8db2ac07e048bd9f9b56e7030a015a 2024-11-15T22:15:58,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/64bb42c47a5b489b8135a424cc48336a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/64bb42c47a5b489b8135a424cc48336a 2024-11-15T22:15:58,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/eea517cc7bb0457eb64519dd8b3c4c45 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/eea517cc7bb0457eb64519dd8b3c4c45 2024-11-15T22:15:58,332 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/recovered.edits/530.seqid, newMaxSeqId=530, maxSeqId=1 2024-11-15T22:15:58,333 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. 2024-11-15T22:15:58,333 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 0cf44e7263c9528c718ed800ce4e85e9: Waiting for close lock at 1731708957202Running coprocessor pre-close hooks at 1731708957202Disabling compacts and flushes for region at 1731708957202Disabling writes for close at 1731708957202Obtaining lock to block concurrent updates at 1731708957202Preparing flush snapshotting stores in 0cf44e7263c9528c718ed800ce4e85e9 at 1731708957202Finished memstore snapshotting TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731708957203 (+1 ms)Flushing stores of TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9. at 1731708957204 (+1 ms)Flushing 0cf44e7263c9528c718ed800ce4e85e9/A: creating writer at 1731708957204Flushing 0cf44e7263c9528c718ed800ce4e85e9/A: appending metadata at 1731708957209 (+5 ms)Flushing 0cf44e7263c9528c718ed800ce4e85e9/A: closing flushed file at 1731708957209Flushing 0cf44e7263c9528c718ed800ce4e85e9/B: creating writer at 1731708957232 (+23 ms)Flushing 0cf44e7263c9528c718ed800ce4e85e9/B: appending metadata at 1731708957239 (+7 ms)Flushing 0cf44e7263c9528c718ed800ce4e85e9/B: closing flushed file at 1731708957239Flushing 0cf44e7263c9528c718ed800ce4e85e9/C: creating writer at 1731708957660 (+421 ms)Flushing 0cf44e7263c9528c718ed800ce4e85e9/C: appending metadata at 1731708957664 (+4 ms)Flushing 0cf44e7263c9528c718ed800ce4e85e9/C: closing flushed file at 1731708957664Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e8d799f: reopening flushed file at 1731708958074 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45657f07: reopening flushed file at 1731708958081 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59de20e2: reopening flushed file at 1731708958096 (+15 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 0cf44e7263c9528c718ed800ce4e85e9 in 901ms, sequenceid=527, compaction requested=false at 1731708958103 (+7 ms)Writing region close event to WAL at 1731708958326 (+223 ms)Running coprocessor post-close hooks at 1731708958333 (+7 ms)Closed at 1731708958333 2024-11-15T22:15:58,336 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:58,337 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=0cf44e7263c9528c718ed800ce4e85e9, regionState=CLOSED 2024-11-15T22:15:58,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:15:58,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-15T22:15:58,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 0cf44e7263c9528c718ed800ce4e85e9, server=e611192d6313,38779,1731708927034 in 1.3100 sec 2024-11-15T22:15:58,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-15T22:15:58,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0cf44e7263c9528c718ed800ce4e85e9, UNASSIGN in 1.3290 sec 2024-11-15T22:15:58,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-15T22:15:58,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3400 sec 2024-11-15T22:15:58,364 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708958363"}]},"ts":"1731708958363"} 2024-11-15T22:15:58,366 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T22:15:58,366 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T22:15:58,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4080 sec 2024-11-15T22:15:59,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T22:15:59,121 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:15:59,125 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T22:15:59,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:15:59,131 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:15:59,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T22:15:59,133 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:15:59,139 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:59,143 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/recovered.edits] 2024-11-15T22:15:59,146 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/0448bd458e25475e9e111c62d240cbbb to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/0448bd458e25475e9e111c62d240cbbb 2024-11-15T22:15:59,148 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/e8c6ed9905634760b02b11d5e6e6f947 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/A/e8c6ed9905634760b02b11d5e6e6f947 2024-11-15T22:15:59,152 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/0845a0415ad443349fdf6634798b7a19 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/0845a0415ad443349fdf6634798b7a19 2024-11-15T22:15:59,154 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/465a47bb1e654ab2b3a1a988196d96ba to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/B/465a47bb1e654ab2b3a1a988196d96ba 2024-11-15T22:15:59,156 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/48f9ac9829224bd8b00c89ed646fa6c7 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/48f9ac9829224bd8b00c89ed646fa6c7 2024-11-15T22:15:59,158 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ac2fa447df3a49e988f7a166bf1cfd4b to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/C/ac2fa447df3a49e988f7a166bf1cfd4b 2024-11-15T22:15:59,161 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/recovered.edits/530.seqid to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9/recovered.edits/530.seqid 2024-11-15T22:15:59,161 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/0cf44e7263c9528c718ed800ce4e85e9 2024-11-15T22:15:59,162 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T22:15:59,166 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:15:59,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-15T22:15:59,176 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T22:15:59,213 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T22:15:59,215 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:15:59,215 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T22:15:59,215 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731708959215"}]},"ts":"9223372036854775807"} 2024-11-15T22:15:59,220 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T22:15:59,220 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0cf44e7263c9528c718ed800ce4e85e9, NAME => 'TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T22:15:59,220 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T22:15:59,220 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731708959220"}]},"ts":"9223372036854775807"} 2024-11-15T22:15:59,223 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T22:15:59,224 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:15:59,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 98 msec 2024-11-15T22:15:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T22:15:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T22:15:59,242 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:15:59,253 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=250 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;e611192d6313:38779-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1052444504_22 at /127.0.0.1:33598 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/e611192d6313:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-556696224_22 at /127.0.0.1:52072 [Waiting for operation #277] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/e611192d6313:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1052444504_22 at /127.0.0.1:56282 [Waiting for operation #181] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-556696224_22 at /127.0.0.1:56020 [Waiting for operation #263] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=859 (was 653) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1412 (was 1301) - AvailableMemoryMB LEAK? - 2024-11-15T22:15:59,262 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=250, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=859, ProcessCount=11, AvailableMemoryMB=1411 2024-11-15T22:15:59,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:15:59,264 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:15:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T22:15:59,267 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T22:15:59,267 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:15:59,267 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-15T22:15:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T22:15:59,268 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T22:15:59,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741954_1130 (size=1061) 2024-11-15T22:15:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T22:15:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T22:15:59,696 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e 2024-11-15T22:15:59,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741955_1131 (size=53) 2024-11-15T22:15:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T22:16:00,105 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:16:00,105 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7b87423906fae6354c3396098e18811c, disabling compactions & flushes 2024-11-15T22:16:00,105 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:00,105 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:00,105 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. after waiting 0 ms 2024-11-15T22:16:00,105 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:00,105 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:00,105 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7b87423906fae6354c3396098e18811c: Waiting for close lock at 1731708960105Disabling compacts and flushes for region at 1731708960105Disabling writes for close at 1731708960105Writing region close event to WAL at 1731708960105Closed at 1731708960105 2024-11-15T22:16:00,107 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T22:16:00,107 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731708960107"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708960107"}]},"ts":"1731708960107"} 2024-11-15T22:16:00,112 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T22:16:00,113 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T22:16:00,114 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708960113"}]},"ts":"1731708960113"} 2024-11-15T22:16:00,116 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T22:16:00,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7b87423906fae6354c3396098e18811c, ASSIGN}] 2024-11-15T22:16:00,118 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7b87423906fae6354c3396098e18811c, ASSIGN 2024-11-15T22:16:00,119 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7b87423906fae6354c3396098e18811c, ASSIGN; state=OFFLINE, location=e611192d6313,38779,1731708927034; forceNewPlan=false, retain=false 2024-11-15T22:16:00,270 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=7b87423906fae6354c3396098e18811c, regionState=OPENING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:16:00,272 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7b87423906fae6354c3396098e18811c, ASSIGN because future has completed 2024-11-15T22:16:00,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:16:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T22:16:00,432 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:00,432 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:16:00,433 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:00,433 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:16:00,433 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:00,433 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:00,434 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:00,436 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:16:00,436 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b87423906fae6354c3396098e18811c columnFamilyName A 2024-11-15T22:16:00,437 DEBUG [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:00,437 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.HStore(327): Store=7b87423906fae6354c3396098e18811c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:16:00,437 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:00,439 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:16:00,439 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b87423906fae6354c3396098e18811c columnFamilyName B 2024-11-15T22:16:00,439 DEBUG [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:00,439 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.HStore(327): Store=7b87423906fae6354c3396098e18811c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:16:00,440 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:00,441 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:16:00,441 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b87423906fae6354c3396098e18811c columnFamilyName C 2024-11-15T22:16:00,441 DEBUG [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:00,441 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.HStore(327): Store=7b87423906fae6354c3396098e18811c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:16:00,442 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:00,442 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:00,442 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c 2024-11-15T22:16:00,443 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c 2024-11-15T22:16:00,444 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:00,444 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:00,444 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:16:00,445 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:00,447 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:16:00,447 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 7b87423906fae6354c3396098e18811c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70964418, jitterRate=0.057452231645584106}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:16:00,447 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:00,448 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 7b87423906fae6354c3396098e18811c: Running coprocessor pre-open hook at 1731708960433Writing region info on filesystem at 1731708960433Initializing all the Stores at 1731708960434 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708960434Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708960434Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708960434Cleaning up temporary data from old regions at 1731708960444 (+10 ms)Running coprocessor post-open hooks at 1731708960447 (+3 ms)Region opened successfully at 1731708960448 (+1 ms) 2024-11-15T22:16:00,449 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., pid=30, masterSystemTime=1731708960425 2024-11-15T22:16:00,451 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:00,451 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:00,451 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=7b87423906fae6354c3396098e18811c, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:16:00,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:16:00,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-15T22:16:00,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 in 181 msec 2024-11-15T22:16:00,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-15T22:16:00,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7b87423906fae6354c3396098e18811c, ASSIGN in 339 msec 2024-11-15T22:16:00,457 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T22:16:00,457 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708960457"}]},"ts":"1731708960457"} 2024-11-15T22:16:00,459 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T22:16:00,460 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T22:16:00,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1960 sec 2024-11-15T22:16:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T22:16:01,412 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:01,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdfec4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:01,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:01,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:01,415 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:01,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:01,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:01,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af5b370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:01,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:01,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:01,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:01,418 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:01,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cedc1ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:01,419 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:01,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:01,424 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T22:16:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is e611192d6313,34969,1731708925722 2024-11-15T22:16:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17eb0dc9 2024-11-15T22:16:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T22:16:01,427 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T22:16:01,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:16:01,433 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:16:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:01,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741956_1132 (size=1097) 2024-11-15T22:16:01,856 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-15T22:16:01,856 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-15T22:16:01,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:16:01,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7b87423906fae6354c3396098e18811c, REOPEN/MOVE}] 2024-11-15T22:16:01,870 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7b87423906fae6354c3396098e18811c, REOPEN/MOVE 2024-11-15T22:16:01,871 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=7b87423906fae6354c3396098e18811c, regionState=CLOSING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:16:01,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7b87423906fae6354c3396098e18811c, REOPEN/MOVE because future has completed 2024-11-15T22:16:01,874 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:16:01,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:16:02,027 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,027 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:16:02,027 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 7b87423906fae6354c3396098e18811c, disabling compactions & flushes 2024-11-15T22:16:02,027 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:02,027 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:02,027 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. after waiting 0 ms 2024-11-15T22:16:02,027 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:02,037 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:16:02,037 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:02,038 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 7b87423906fae6354c3396098e18811c: Waiting for close lock at 1731708962027Running coprocessor pre-close hooks at 1731708962027Disabling compacts and flushes for region at 1731708962027Disabling writes for close at 1731708962027Writing region close event to WAL at 1731708962028 (+1 ms)Running coprocessor post-close hooks at 1731708962037 (+9 ms)Closed at 1731708962037 2024-11-15T22:16:02,038 WARN [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 7b87423906fae6354c3396098e18811c to self. 2024-11-15T22:16:02,043 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,044 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=7b87423906fae6354c3396098e18811c, regionState=CLOSED 2024-11-15T22:16:02,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:16:02,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-15T22:16:02,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 in 175 msec 2024-11-15T22:16:02,052 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7b87423906fae6354c3396098e18811c, REOPEN/MOVE; state=CLOSED, location=e611192d6313,38779,1731708927034; forceNewPlan=false, retain=true 2024-11-15T22:16:02,202 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=7b87423906fae6354c3396098e18811c, regionState=OPENING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:16:02,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7b87423906fae6354c3396098e18811c, REOPEN/MOVE because future has completed 2024-11-15T22:16:02,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:16:02,360 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:02,360 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:16:02,360 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,360 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:16:02,360 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,360 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,364 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,365 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:16:02,371 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b87423906fae6354c3396098e18811c columnFamilyName A 2024-11-15T22:16:02,373 DEBUG [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:02,373 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.HStore(327): Store=7b87423906fae6354c3396098e18811c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:16:02,374 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,375 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:16:02,375 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b87423906fae6354c3396098e18811c columnFamilyName B 2024-11-15T22:16:02,375 DEBUG [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:02,375 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.HStore(327): Store=7b87423906fae6354c3396098e18811c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:16:02,375 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,376 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:16:02,376 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b87423906fae6354c3396098e18811c columnFamilyName C 2024-11-15T22:16:02,376 DEBUG [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:02,377 INFO [StoreOpener-7b87423906fae6354c3396098e18811c-1 {}] regionserver.HStore(327): Store=7b87423906fae6354c3396098e18811c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:16:02,377 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:02,377 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,378 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,379 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,380 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,380 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,381 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:16:02,383 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,384 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 7b87423906fae6354c3396098e18811c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62707356, jitterRate=-0.06558758020401001}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:16:02,384 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,385 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 7b87423906fae6354c3396098e18811c: Running coprocessor pre-open hook at 1731708962361Writing region info on filesystem at 1731708962361Initializing all the Stores at 1731708962361Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708962361Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708962362 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708962362Cleaning up temporary data from old regions at 1731708962380 (+18 ms)Running coprocessor post-open hooks at 1731708962384 (+4 ms)Region opened successfully at 1731708962385 (+1 ms) 2024-11-15T22:16:02,385 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., pid=35, masterSystemTime=1731708962356 2024-11-15T22:16:02,387 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:02,387 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:02,387 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=7b87423906fae6354c3396098e18811c, regionState=OPEN, openSeqNum=5, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:16:02,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:16:02,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-15T22:16:02,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 in 186 msec 2024-11-15T22:16:02,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-15T22:16:02,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7b87423906fae6354c3396098e18811c, REOPEN/MOVE in 523 msec 2024-11-15T22:16:02,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-15T22:16:02,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 535 msec 2024-11-15T22:16:02,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 964 msec 2024-11-15T22:16:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T22:16:02,404 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:02,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60dba4ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:02,405 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:02,406 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c333d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:02,408 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bf142fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20c00921, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,410 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:02,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:02,411 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efd2437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:02,413 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:02,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61bb2348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76b1860b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:02,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:02,416 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c3f7b04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:02,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:02,419 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21dcea60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22aa04a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:02,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:02,421 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:02,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f3e5ef7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:02,423 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:02,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@616924c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52903b3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:02,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:02,426 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:02,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:02,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:02,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1283adf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:02,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:02,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:02,428 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:02,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@756ad3ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0d456e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,430 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:02,430 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:02,431 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18dc78e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:02,433 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:02,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f782ca4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43fbca19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:02,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:02,437 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:02,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:02,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:02,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761a8232, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:02,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:02,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:02,439 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:02,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7203a827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c562e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,444 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:02,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:02,445 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed9cd7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:02,447 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:02,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7825eb66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdf07c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:02,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:02,450 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:02,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:02,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:02,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26fc9d58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:02,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:02,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:02,452 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@359e56d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:02,454 DEBUG [Thread-666 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:02,454 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:02,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:02,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:02,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:02,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:02,456 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:02,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:02,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:02,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-15T22:16:02,459 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T22:16:02,460 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:02,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:02,465 DEBUG [Thread-669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:02,466 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:02,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:02,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:02,467 DEBUG [Thread-671 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:02,468 DEBUG [Thread-671 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:02,470 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:02,470 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:02,477 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:02,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:02,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:02,488 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:02,489 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:02,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:02,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:02,490 DEBUG [Thread-658 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:02,490 DEBUG [Thread-658 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:02,491 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:02,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:02,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:02,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:02,495 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:02,495 DEBUG [Thread-664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:02,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:02,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:02,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:02,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:02,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T22:16:02,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:02,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:02,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:02,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:02,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:02,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:02,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T22:16:02,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111595f9c67bdd5c426fb4f77438958d7fc3_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708962505/Put/seqid=0 2024-11-15T22:16:02,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741957_1133 (size=14594) 2024-11-15T22:16:02,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T22:16:02,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:02,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:02,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:02,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:02,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:02,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709022627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:02,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708962569, max timestamp=1731708962626Num uniques 859; 2024-11-15T22:16:02,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708962569, max timestamp=1731708962626Num uniques 859; 2024-11-15T22:16:02,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708962569, max timestamp=1731708962626Num uniques 859; 2024-11-15T22:16:02,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:16:02,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:16:02,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:16:02,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709022699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:02,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731708962684, max timestamp=1731708962707Num uniques 1018; 2024-11-15T22:16:02,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-15T22:16:02,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731708962688, max timestamp=1731708962707Num uniques 1018; 2024-11-15T22:16:02,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731708962684, max timestamp=1731708962707Num uniques 1018; 2024-11-15T22:16:02,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-15T22:16:02,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-15T22:16:02,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T22:16:02,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:02,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:02,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:02,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:02,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:02,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709022762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:02,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709022764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:02,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T22:16:02,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:02,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709022766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:02,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:02,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709022780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:02,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:02,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709022823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:02,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709022885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:02,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:02,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709022893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:02,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:02,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:02,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709022904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709022893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:02,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T22:16:02,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:02,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:02,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:02,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:02,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,011 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:03,018 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111595f9c67bdd5c426fb4f77438958d7fc3_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111595f9c67bdd5c426fb4f77438958d7fc3_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:03,019 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/c3bf897b52cc4e4db55585a903806cf5, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:03,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/c3bf897b52cc4e4db55585a903806cf5 is 175, key is test_row_0/A:col10/1731708962505/Put/seqid=0 2024-11-15T22:16:03,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741958_1134 (size=39549) 2024-11-15T22:16:03,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709023042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:03,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T22:16:03,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:03,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:03,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:03,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T22:16:03,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709023112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:03,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709023121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:03,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709023122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:03,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709023123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:03,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:03,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T22:16:03,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:03,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:03,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:03,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709023351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:03,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T22:16:03,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:03,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:03,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:03,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709023422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:03,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709023432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:03,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709023432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:03,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709023432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:03,444 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/c3bf897b52cc4e4db55585a903806cf5 2024-11-15T22:16:03,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/67975ef7f21e4f9e81ae4685353ab9dc is 50, key is test_row_0/B:col10/1731708962505/Put/seqid=0 2024-11-15T22:16:03,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741959_1135 (size=12001) 2024-11-15T22:16:03,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/67975ef7f21e4f9e81ae4685353ab9dc 2024-11-15T22:16:03,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T22:16:03,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:03,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:03,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:03,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/566d17d1c09e4a99911cbfeee3d8c3f4 is 50, key is test_row_0/C:col10/1731708962505/Put/seqid=0 2024-11-15T22:16:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T22:16:03,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741960_1136 (size=12001) 2024-11-15T22:16:03,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T22:16:03,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:03,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:03,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:03,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T22:16:03,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:03,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:03,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:03,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:03,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709023872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:03,892 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T22:16:03,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709023953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709023953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:03,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709023953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:03,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709023953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:03,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:03,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:04,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T22:16:04,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:04,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:04,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:04,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:04,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:04,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/566d17d1c09e4a99911cbfeee3d8c3f4 2024-11-15T22:16:04,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/c3bf897b52cc4e4db55585a903806cf5 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c3bf897b52cc4e4db55585a903806cf5 2024-11-15T22:16:04,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c3bf897b52cc4e4db55585a903806cf5, entries=200, sequenceid=14, filesize=38.6 K 2024-11-15T22:16:04,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/67975ef7f21e4f9e81ae4685353ab9dc as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/67975ef7f21e4f9e81ae4685353ab9dc 2024-11-15T22:16:04,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/67975ef7f21e4f9e81ae4685353ab9dc, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T22:16:04,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/566d17d1c09e4a99911cbfeee3d8c3f4 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/566d17d1c09e4a99911cbfeee3d8c3f4 2024-11-15T22:16:04,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/566d17d1c09e4a99911cbfeee3d8c3f4, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T22:16:04,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.38 KB/136582 for 7b87423906fae6354c3396098e18811c in 1576ms, sequenceid=14, compaction requested=false 2024-11-15T22:16:04,087 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T22:16:04,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:04,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T22:16:04,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:04,155 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=133.38 KB heapSize=372.94 KB 2024-11-15T22:16:04,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:04,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:04,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:04,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:04,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:04,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:04,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150a2cee14349d44109cf1c29c3a60ee94_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708962775/Put/seqid=0 2024-11-15T22:16:04,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741961_1137 (size=12154) 2024-11-15T22:16:04,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,252 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150a2cee14349d44109cf1c29c3a60ee94_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150a2cee14349d44109cf1c29c3a60ee94_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:04,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/fe583f7cfaab451f854d60b196277b1d, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:04,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/fe583f7cfaab451f854d60b196277b1d is 175, key is test_row_0/A:col10/1731708962775/Put/seqid=0 2024-11-15T22:16:04,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741962_1138 (size=30955) 2024-11-15T22:16:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T22:16:04,707 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=66, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/fe583f7cfaab451f854d60b196277b1d 2024-11-15T22:16:04,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/73dfb23a3d6f4bdfb9831e9c82d3e381 is 50, key is test_row_0/B:col10/1731708962775/Put/seqid=0 2024-11-15T22:16:04,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741963_1139 (size=12001) 2024-11-15T22:16:04,775 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/73dfb23a3d6f4bdfb9831e9c82d3e381 2024-11-15T22:16:04,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/4ccb2dbaf5a4474ea66970dc27f94632 is 50, key is test_row_0/C:col10/1731708962775/Put/seqid=0 2024-11-15T22:16:04,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741964_1140 (size=12001) 2024-11-15T22:16:04,859 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.46 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/4ccb2dbaf5a4474ea66970dc27f94632 2024-11-15T22:16:04,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/fe583f7cfaab451f854d60b196277b1d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/fe583f7cfaab451f854d60b196277b1d 2024-11-15T22:16:04,885 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/fe583f7cfaab451f854d60b196277b1d, entries=150, sequenceid=66, filesize=30.2 K 2024-11-15T22:16:04,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/73dfb23a3d6f4bdfb9831e9c82d3e381 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/73dfb23a3d6f4bdfb9831e9c82d3e381 2024-11-15T22:16:04,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,897 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/73dfb23a3d6f4bdfb9831e9c82d3e381, entries=150, sequenceid=66, filesize=11.7 K 2024-11-15T22:16:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/4ccb2dbaf5a4474ea66970dc27f94632 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/4ccb2dbaf5a4474ea66970dc27f94632 2024-11-15T22:16:04,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,906 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/4ccb2dbaf5a4474ea66970dc27f94632, entries=150, sequenceid=66, filesize=11.7 K 2024-11-15T22:16:04,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,908 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~133.38 KB/136582, heapSize ~372.89 KB/381840, currentSize=0 B/0 for 7b87423906fae6354c3396098e18811c in 752ms, sequenceid=66, compaction requested=false 2024-11-15T22:16:04,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:04,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:04,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-15T22:16:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-15T22:16:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,912 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-15T22:16:04,912 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4490 sec 2024-11-15T22:16:04,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.4570 sec 2024-11-15T22:16:04,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T22:16:05,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:05,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:05,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:05,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:05,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:05,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153533307c65674150b59d6a60cb95c89c_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708965033/Put/seqid=0 2024-11-15T22:16:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741965_1141 (size=21918) 2024-11-15T22:16:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,102 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,108 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153533307c65674150b59d6a60cb95c89c_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153533307c65674150b59d6a60cb95c89c_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:05,120 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/c41165631cdc46588fa02059fb8cfaae, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:05,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/c41165631cdc46588fa02059fb8cfaae is 175, key is test_row_0/A:col10/1731708965033/Put/seqid=0 2024-11-15T22:16:05,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731708965119, max timestamp=1731708965131Num uniques 859; 2024-11-15T22:16:05,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731708965119, max timestamp=1731708965131Num uniques 859; 2024-11-15T22:16:05,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:16:05,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:16:05,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731708965119, max timestamp=1731708965131Num uniques 859; 2024-11-15T22:16:05,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:16:05,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741966_1142 (size=65323) 2024-11-15T22:16:05,157 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/c41165631cdc46588fa02059fb8cfaae 2024-11-15T22:16:05,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/0a7f64045def449aa44a4085c15638bd is 50, key is test_row_0/B:col10/1731708965033/Put/seqid=0 2024-11-15T22:16:05,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709025179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709025179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709025185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709025189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709025198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741967_1143 (size=12001) 2024-11-15T22:16:05,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709025302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709025302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709025311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709025311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709025322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709025512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709025512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709025521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709025521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709025542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/0a7f64045def449aa44a4085c15638bd 2024-11-15T22:16:05,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/1e6a5aea32e2442c94c51f136d2bc726 is 50, key is test_row_0/C:col10/1731708965033/Put/seqid=0 2024-11-15T22:16:05,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741968_1144 (size=12001) 2024-11-15T22:16:05,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/1e6a5aea32e2442c94c51f136d2bc726 2024-11-15T22:16:05,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/c41165631cdc46588fa02059fb8cfaae as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c41165631cdc46588fa02059fb8cfaae 2024-11-15T22:16:05,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c41165631cdc46588fa02059fb8cfaae, entries=350, sequenceid=76, filesize=63.8 K 2024-11-15T22:16:05,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/0a7f64045def449aa44a4085c15638bd as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/0a7f64045def449aa44a4085c15638bd 2024-11-15T22:16:05,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/0a7f64045def449aa44a4085c15638bd, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T22:16:05,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/1e6a5aea32e2442c94c51f136d2bc726 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/1e6a5aea32e2442c94c51f136d2bc726 2024-11-15T22:16:05,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/1e6a5aea32e2442c94c51f136d2bc726, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T22:16:05,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7b87423906fae6354c3396098e18811c in 673ms, sequenceid=76, compaction requested=true 2024-11-15T22:16:05,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:05,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:05,710 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:05,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:05,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:05,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:05,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:05,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:05,710 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:05,711 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:05,711 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:05,712 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/A is initiating minor compaction (all files) 2024-11-15T22:16:05,712 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/B is initiating minor compaction (all files) 2024-11-15T22:16:05,712 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/A in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:05,712 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/B in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:05,712 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c3bf897b52cc4e4db55585a903806cf5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/fe583f7cfaab451f854d60b196277b1d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c41165631cdc46588fa02059fb8cfaae] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=132.6 K 2024-11-15T22:16:05,712 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/67975ef7f21e4f9e81ae4685353ab9dc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/73dfb23a3d6f4bdfb9831e9c82d3e381, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/0a7f64045def449aa44a4085c15638bd] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=35.2 K 2024-11-15T22:16:05,712 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:05,712 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c3bf897b52cc4e4db55585a903806cf5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/fe583f7cfaab451f854d60b196277b1d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c41165631cdc46588fa02059fb8cfaae] 2024-11-15T22:16:05,713 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 67975ef7f21e4f9e81ae4685353ab9dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731708962503 2024-11-15T22:16:05,713 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3bf897b52cc4e4db55585a903806cf5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731708962498 2024-11-15T22:16:05,713 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 73dfb23a3d6f4bdfb9831e9c82d3e381, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731708962758 2024-11-15T22:16:05,714 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe583f7cfaab451f854d60b196277b1d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731708962758 2024-11-15T22:16:05,714 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a7f64045def449aa44a4085c15638bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731708965013 2024-11-15T22:16:05,714 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting c41165631cdc46588fa02059fb8cfaae, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731708965008 2024-11-15T22:16:05,739 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:05,740 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/246cb621969444888ecfba228e7d9c87 is 50, key is test_row_0/B:col10/1731708965033/Put/seqid=0 2024-11-15T22:16:05,745 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:05,765 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111509aef773f6164194b0db5a0b12b25963_7b87423906fae6354c3396098e18811c store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:05,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741969_1145 (size=12104) 2024-11-15T22:16:05,773 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111509aef773f6164194b0db5a0b12b25963_7b87423906fae6354c3396098e18811c, store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:05,773 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111509aef773f6164194b0db5a0b12b25963_7b87423906fae6354c3396098e18811c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:05,778 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/246cb621969444888ecfba228e7d9c87 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/246cb621969444888ecfba228e7d9c87 2024-11-15T22:16:05,787 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b87423906fae6354c3396098e18811c/B of 7b87423906fae6354c3396098e18811c into 246cb621969444888ecfba228e7d9c87(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:05,787 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:05,787 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/B, priority=13, startTime=1731708965710; duration=0sec 2024-11-15T22:16:05,787 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:05,787 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:B 2024-11-15T22:16:05,787 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:05,789 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:05,789 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/C is initiating minor compaction (all files) 2024-11-15T22:16:05,789 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/C in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:05,789 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/566d17d1c09e4a99911cbfeee3d8c3f4, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/4ccb2dbaf5a4474ea66970dc27f94632, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/1e6a5aea32e2442c94c51f136d2bc726] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=35.2 K 2024-11-15T22:16:05,790 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 566d17d1c09e4a99911cbfeee3d8c3f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731708962503 2024-11-15T22:16:05,790 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ccb2dbaf5a4474ea66970dc27f94632, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731708962758 2024-11-15T22:16:05,791 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e6a5aea32e2442c94c51f136d2bc726, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731708965013 2024-11-15T22:16:05,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741970_1146 (size=4469) 2024-11-15T22:16:05,806 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#A#compaction#127 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:05,806 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/9897286d002b402f990ab4d7f3bc63bd is 175, key is test_row_0/A:col10/1731708965033/Put/seqid=0 2024-11-15T22:16:05,808 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#C#compaction#128 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:05,809 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/7bf853aca4ca428493e6feff034e06fa is 50, key is test_row_0/C:col10/1731708965033/Put/seqid=0 2024-11-15T22:16:05,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-15T22:16:05,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:05,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:05,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:05,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:05,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:05,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:05,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741971_1147 (size=31058) 2024-11-15T22:16:05,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741972_1148 (size=12104) 2024-11-15T22:16:05,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709025842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709025849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709025844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709025851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,862 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/7bf853aca4ca428493e6feff034e06fa as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/7bf853aca4ca428493e6feff034e06fa 2024-11-15T22:16:05,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709025861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,876 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b87423906fae6354c3396098e18811c/C of 7b87423906fae6354c3396098e18811c into 7bf853aca4ca428493e6feff034e06fa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:05,876 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:05,876 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/C, priority=13, startTime=1731708965710; duration=0sec 2024-11-15T22:16:05,876 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:05,876 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:C 2024-11-15T22:16:05,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115498f2511303945128287e36de0e0d99b_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708965826/Put/seqid=0 2024-11-15T22:16:05,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741973_1149 (size=14594) 2024-11-15T22:16:05,892 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:05,899 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115498f2511303945128287e36de0e0d99b_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115498f2511303945128287e36de0e0d99b_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:05,901 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/ee0907f4d7fb49ed9f9fb73235ed2c1f, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:05,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/ee0907f4d7fb49ed9f9fb73235ed2c1f is 175, key is test_row_0/A:col10/1731708965826/Put/seqid=0 2024-11-15T22:16:05,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741974_1150 (size=39549) 2024-11-15T22:16:05,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709025962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709025962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709025962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:06,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709026172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709026172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709026172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:06,264 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/9897286d002b402f990ab4d7f3bc63bd as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/9897286d002b402f990ab4d7f3bc63bd 2024-11-15T22:16:06,276 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b87423906fae6354c3396098e18811c/A of 7b87423906fae6354c3396098e18811c into 9897286d002b402f990ab4d7f3bc63bd(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:06,277 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:06,277 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/A, priority=13, startTime=1731708965709; duration=0sec 2024-11-15T22:16:06,277 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:06,278 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:A 2024-11-15T22:16:06,325 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=114, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/ee0907f4d7fb49ed9f9fb73235ed2c1f 2024-11-15T22:16:06,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/d0a8df407ba445cc9e34f3e27a0f15e8 is 50, key is test_row_0/B:col10/1731708965826/Put/seqid=0 2024-11-15T22:16:06,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741975_1151 (size=12001) 2024-11-15T22:16:06,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709026361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:06,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:06,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709026382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:06,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:06,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709026485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:06,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709026486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:06,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709026486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T22:16:06,621 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:06,622 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-15T22:16:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T22:16:06,624 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:06,626 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:06,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T22:16:06,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/d0a8df407ba445cc9e34f3e27a0f15e8 2024-11-15T22:16:06,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T22:16:06,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:06,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:06,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:06,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:06,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:06,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/9803126df3304ef3949aaae915b2334b is 50, key is test_row_0/C:col10/1731708965826/Put/seqid=0 2024-11-15T22:16:06,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741976_1152 (size=12001) 2024-11-15T22:16:06,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/9803126df3304ef3949aaae915b2334b 2024-11-15T22:16:06,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/ee0907f4d7fb49ed9f9fb73235ed2c1f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ee0907f4d7fb49ed9f9fb73235ed2c1f 2024-11-15T22:16:06,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ee0907f4d7fb49ed9f9fb73235ed2c1f, entries=200, sequenceid=114, filesize=38.6 K 2024-11-15T22:16:06,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/d0a8df407ba445cc9e34f3e27a0f15e8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/d0a8df407ba445cc9e34f3e27a0f15e8 2024-11-15T22:16:06,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/d0a8df407ba445cc9e34f3e27a0f15e8, entries=150, sequenceid=114, filesize=11.7 K 2024-11-15T22:16:06,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/9803126df3304ef3949aaae915b2334b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/9803126df3304ef3949aaae915b2334b 2024-11-15T22:16:06,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/9803126df3304ef3949aaae915b2334b, entries=150, sequenceid=114, filesize=11.7 K 2024-11-15T22:16:06,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 7b87423906fae6354c3396098e18811c in 1013ms, sequenceid=114, compaction requested=false 2024-11-15T22:16:06,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:06,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T22:16:06,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:06,933 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-15T22:16:06,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:06,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:06,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:06,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:06,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:06,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T22:16:06,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115352d42dfb2b34fac967f1ba15c0cd473_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708965833/Put/seqid=0 2024-11-15T22:16:06,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741977_1153 (size=12154) 2024-11-15T22:16:06,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:06,996 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115352d42dfb2b34fac967f1ba15c0cd473_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115352d42dfb2b34fac967f1ba15c0cd473_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:06,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/ab8c57fb7db74d02b0d01a54a3ba1dc8, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:06,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/ab8c57fb7db74d02b0d01a54a3ba1dc8 is 175, key is test_row_0/A:col10/1731708965833/Put/seqid=0 2024-11-15T22:16:07,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:07,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741978_1154 (size=30955) 2024-11-15T22:16:07,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731708967036, max timestamp=1731708967045Num uniques 859; 2024-11-15T22:16:07,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T22:16:07,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731708967036, max timestamp=1731708967045Num uniques 859; 2024-11-15T22:16:07,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T22:16:07,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731708967036, max timestamp=1731708967045Num uniques 859; 2024-11-15T22:16:07,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T22:16:07,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709027099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:07,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731708967060, max timestamp=1731708967104Num uniques 1118; 2024-11-15T22:16:07,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731708967060, max timestamp=1731708967104Num uniques 1009; 2024-11-15T22:16:07,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-15T22:16:07,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-15T22:16:07,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731708967060, max timestamp=1731708967104Num uniques 1009; 2024-11-15T22:16:07,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-15T22:16:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:07,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709027173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:07,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709027175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:07,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709027221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T22:16:07,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709027291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:07,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709027291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:07,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709027372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:07,375 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in 7b87423906fae6354c3396098e18811c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:16:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:07,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709027402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:07,406 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_0' in 7b87423906fae6354c3396098e18811c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:16:07,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:07,417 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=125, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/ab8c57fb7db74d02b0d01a54a3ba1dc8 2024-11-15T22:16:07,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709027432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:07,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:07,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/b531dd4b8679422aa907a2a908869324 is 50, key is test_row_0/B:col10/1731708965833/Put/seqid=0 2024-11-15T22:16:07,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741979_1155 (size=12001) 2024-11-15T22:16:07,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709027501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:07,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709027502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:07,517 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/b531dd4b8679422aa907a2a908869324 2024-11-15T22:16:07,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/9908bf093f734cd0a58666aecdeedb00 is 50, key is test_row_0/C:col10/1731708965833/Put/seqid=0 2024-11-15T22:16:07,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741980_1156 (size=12001) 2024-11-15T22:16:07,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:07,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709027752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:07,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T22:16:07,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:07,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709027833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:07,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:07,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709027833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:07,982 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/9908bf093f734cd0a58666aecdeedb00 2024-11-15T22:16:08,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/ab8c57fb7db74d02b0d01a54a3ba1dc8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ab8c57fb7db74d02b0d01a54a3ba1dc8 2024-11-15T22:16:08,034 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ab8c57fb7db74d02b0d01a54a3ba1dc8, entries=150, sequenceid=125, filesize=30.2 K 2024-11-15T22:16:08,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/b531dd4b8679422aa907a2a908869324 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b531dd4b8679422aa907a2a908869324 2024-11-15T22:16:08,083 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b531dd4b8679422aa907a2a908869324, entries=150, sequenceid=125, filesize=11.7 K 2024-11-15T22:16:08,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/9908bf093f734cd0a58666aecdeedb00 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/9908bf093f734cd0a58666aecdeedb00 2024-11-15T22:16:08,099 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/9908bf093f734cd0a58666aecdeedb00, entries=150, sequenceid=125, filesize=11.7 K 2024-11-15T22:16:08,108 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 7b87423906fae6354c3396098e18811c in 1175ms, sequenceid=125, compaction requested=true 2024-11-15T22:16:08,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:08,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:08,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-15T22:16:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-15T22:16:08,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-15T22:16:08,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4980 sec 2024-11-15T22:16:08,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.5410 sec 2024-11-15T22:16:08,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731708967149, max timestamp=1731708968277Num uniques 1077; 2024-11-15T22:16:08,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-15T22:16:08,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=107.34 KB heapSize=301.52 KB 2024-11-15T22:16:08,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:08,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:08,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:08,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T22:16:08,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:08,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T22:16:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:08,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115856a3032bd0e4834b3a2edd9a7eb312f_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708967170/Put/seqid=0 2024-11-15T22:16:08,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741981_1157 (size=14794) 2024-11-15T22:16:08,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709028401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:08,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709028408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:08,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709028414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:08,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709028521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:08,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709028522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:08,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709028527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:08,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:08,759 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:08,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:08,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709028752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:08,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:08,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709028752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:08,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:08,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709028762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:08,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T22:16:08,776 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:08,776 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115856a3032bd0e4834b3a2edd9a7eb312f_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115856a3032bd0e4834b3a2edd9a7eb312f_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:08,780 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-15T22:16:08,785 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/06e31d4f67134ea5927ce2f41728b5ef, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:08,785 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:08,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/06e31d4f67134ea5927ce2f41728b5ef is 175, key is test_row_0/A:col10/1731708967170/Put/seqid=0 2024-11-15T22:16:08,787 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:08,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:08,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T22:16:08,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741982_1158 (size=39749) 2024-11-15T22:16:08,853 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/06e31d4f67134ea5927ce2f41728b5ef 2024-11-15T22:16:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T22:16:08,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/0b1bb52e8ba7468d84953ac090dd22ae is 50, key is test_row_0/B:col10/1731708967170/Put/seqid=0 2024-11-15T22:16:08,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T22:16:08,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:08,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:08,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:08,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:08,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:08,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741983_1159 (size=12151) 2024-11-15T22:16:09,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709029073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:09,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709029073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:09,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:09,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709029084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:09,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T22:16:09,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T22:16:09,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:09,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:09,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:09,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:09,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:09,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T22:16:09,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:09,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:09,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:09,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:09,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:09,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/0b1bb52e8ba7468d84953ac090dd22ae 2024-11-15T22:16:09,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709029394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:09,399 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in 7b87423906fae6354c3396098e18811c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:16:09,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T22:16:09,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/8aff3b1984c64dfaa433e778c8f0fcd4 is 50, key is test_row_0/C:col10/1731708967170/Put/seqid=0 2024-11-15T22:16:09,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T22:16:09,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:09,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:09,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:09,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:09,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709029436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:09,441 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_0' in 7b87423906fae6354c3396098e18811c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:16:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:09,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741984_1160 (size=12151) 2024-11-15T22:16:09,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/8aff3b1984c64dfaa433e778c8f0fcd4 2024-11-15T22:16:09,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/06e31d4f67134ea5927ce2f41728b5ef as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/06e31d4f67134ea5927ce2f41728b5ef 2024-11-15T22:16:09,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/06e31d4f67134ea5927ce2f41728b5ef, entries=200, sequenceid=180, filesize=38.8 K 2024-11-15T22:16:09,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/0b1bb52e8ba7468d84953ac090dd22ae as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/0b1bb52e8ba7468d84953ac090dd22ae 2024-11-15T22:16:09,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/0b1bb52e8ba7468d84953ac090dd22ae, entries=150, sequenceid=180, filesize=11.9 K 2024-11-15T22:16:09,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/8aff3b1984c64dfaa433e778c8f0fcd4 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/8aff3b1984c64dfaa433e778c8f0fcd4 2024-11-15T22:16:09,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709029581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:09,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/8aff3b1984c64dfaa433e778c8f0fcd4, entries=150, sequenceid=180, filesize=11.9 K 2024-11-15T22:16:09,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709029593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:09,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~301.47 KB/308704, currentSize=73.80 KB/75570 for 7b87423906fae6354c3396098e18811c in 1318ms, sequenceid=180, compaction requested=true 2024-11-15T22:16:09,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:09,601 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:16:09,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:09,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:09,601 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:16:09,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:09,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:09,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:09,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:09,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T22:16:09,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:09,603 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T22:16:09,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:09,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:09,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:09,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:09,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:09,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:09,610 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141311 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:16:09,610 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/A is initiating minor compaction (all files) 2024-11-15T22:16:09,610 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:16:09,610 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/B is initiating minor compaction (all files) 2024-11-15T22:16:09,610 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/A in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:09,610 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/B in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:09,610 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/9897286d002b402f990ab4d7f3bc63bd, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ee0907f4d7fb49ed9f9fb73235ed2c1f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ab8c57fb7db74d02b0d01a54a3ba1dc8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/06e31d4f67134ea5927ce2f41728b5ef] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=138.0 K 2024-11-15T22:16:09,610 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/246cb621969444888ecfba228e7d9c87, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/d0a8df407ba445cc9e34f3e27a0f15e8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b531dd4b8679422aa907a2a908869324, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/0b1bb52e8ba7468d84953ac090dd22ae] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=47.1 K 2024-11-15T22:16:09,610 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:09,610 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/9897286d002b402f990ab4d7f3bc63bd, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ee0907f4d7fb49ed9f9fb73235ed2c1f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ab8c57fb7db74d02b0d01a54a3ba1dc8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/06e31d4f67134ea5927ce2f41728b5ef] 2024-11-15T22:16:09,612 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9897286d002b402f990ab4d7f3bc63bd, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731708965013 2024-11-15T22:16:09,612 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 246cb621969444888ecfba228e7d9c87, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731708965013 2024-11-15T22:16:09,614 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a8df407ba445cc9e34f3e27a0f15e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731708965183 2024-11-15T22:16:09,616 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee0907f4d7fb49ed9f9fb73235ed2c1f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731708965173 2024-11-15T22:16:09,618 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting b531dd4b8679422aa907a2a908869324, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731708965833 2024-11-15T22:16:09,619 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b1bb52e8ba7468d84953ac090dd22ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731708967170 2024-11-15T22:16:09,619 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab8c57fb7db74d02b0d01a54a3ba1dc8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731708965833 2024-11-15T22:16:09,620 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06e31d4f67134ea5927ce2f41728b5ef, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731708967149 2024-11-15T22:16:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:09,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:09,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ac049f44ae77418c8119be151ec3f590_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708968388/Put/seqid=0 2024-11-15T22:16:09,685 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:09,703 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#B#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:09,704 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/8bf0205f09d34ae59850d0a1050e9ce7 is 50, key is test_row_0/B:col10/1731708967170/Put/seqid=0 2024-11-15T22:16:09,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741985_1161 (size=12304) 2024-11-15T22:16:09,737 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411154c6fe0ef02cf4b5793e34a01ca771aeb_7b87423906fae6354c3396098e18811c store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:09,741 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411154c6fe0ef02cf4b5793e34a01ca771aeb_7b87423906fae6354c3396098e18811c, store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:09,741 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154c6fe0ef02cf4b5793e34a01ca771aeb_7b87423906fae6354c3396098e18811c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:09,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741986_1162 (size=12391) 2024-11-15T22:16:09,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741987_1163 (size=4469) 2024-11-15T22:16:09,785 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/8bf0205f09d34ae59850d0a1050e9ce7 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/8bf0205f09d34ae59850d0a1050e9ce7 2024-11-15T22:16:09,793 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#A#compaction#139 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:09,794 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/ad186fbb88434bd39cb1009dd77c0601 is 175, key is test_row_0/A:col10/1731708967170/Put/seqid=0 2024-11-15T22:16:09,810 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7b87423906fae6354c3396098e18811c/B of 7b87423906fae6354c3396098e18811c into 8bf0205f09d34ae59850d0a1050e9ce7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:09,810 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:09,810 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/B, priority=12, startTime=1731708969601; duration=0sec 2024-11-15T22:16:09,810 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:09,810 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:B 2024-11-15T22:16:09,810 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:16:09,813 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:16:09,814 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/C is initiating minor compaction (all files) 2024-11-15T22:16:09,814 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/C in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:09,818 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/7bf853aca4ca428493e6feff034e06fa, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/9803126df3304ef3949aaae915b2334b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/9908bf093f734cd0a58666aecdeedb00, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/8aff3b1984c64dfaa433e778c8f0fcd4] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=47.1 K 2024-11-15T22:16:09,825 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf853aca4ca428493e6feff034e06fa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731708965013 2024-11-15T22:16:09,825 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 9803126df3304ef3949aaae915b2334b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731708965183 2024-11-15T22:16:09,826 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 9908bf093f734cd0a58666aecdeedb00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731708965833 2024-11-15T22:16:09,836 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aff3b1984c64dfaa433e778c8f0fcd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731708967170 2024-11-15T22:16:09,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741988_1164 (size=31345) 2024-11-15T22:16:09,867 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:09,868 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/4297f1cb14ba47f294c15facf141e7ac is 50, key is test_row_0/C:col10/1731708967170/Put/seqid=0 2024-11-15T22:16:09,869 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/ad186fbb88434bd39cb1009dd77c0601 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ad186fbb88434bd39cb1009dd77c0601 2024-11-15T22:16:09,890 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7b87423906fae6354c3396098e18811c/A of 7b87423906fae6354c3396098e18811c into ad186fbb88434bd39cb1009dd77c0601(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:09,890 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:09,890 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/A, priority=12, startTime=1731708969600; duration=0sec 2024-11-15T22:16:09,890 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:09,891 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:A 2024-11-15T22:16:09,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709029897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:09,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:09,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:09,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T22:16:09,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741989_1165 (size=12391) 2024-11-15T22:16:09,951 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/4297f1cb14ba47f294c15facf141e7ac as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/4297f1cb14ba47f294c15facf141e7ac 2024-11-15T22:16:09,989 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7b87423906fae6354c3396098e18811c/C of 7b87423906fae6354c3396098e18811c into 4297f1cb14ba47f294c15facf141e7ac(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:09,989 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:09,989 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/C, priority=12, startTime=1731708969601; duration=0sec 2024-11-15T22:16:09,989 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:09,989 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:C 2024-11-15T22:16:10,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709030031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:10,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:10,162 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ac049f44ae77418c8119be151ec3f590_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ac049f44ae77418c8119be151ec3f590_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:10,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/8287d56eddcf46ee91b6d7263c171cca, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:10,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/8287d56eddcf46ee91b6d7263c171cca is 175, key is test_row_0/A:col10/1731708968388/Put/seqid=0 2024-11-15T22:16:10,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741990_1166 (size=31105) 2024-11-15T22:16:10,222 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/8287d56eddcf46ee91b6d7263c171cca 2024-11-15T22:16:10,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709030241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:10,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/70eb828b120347da99bb15f39d9b3f28 is 50, key is test_row_0/B:col10/1731708968388/Put/seqid=0 2024-11-15T22:16:10,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741991_1167 (size=12151) 2024-11-15T22:16:10,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709030564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:10,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709030602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:10,603 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in 7b87423906fae6354c3396098e18811c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:16:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:10,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709030602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:10,613 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in 7b87423906fae6354c3396098e18811c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:16:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:10,726 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/70eb828b120347da99bb15f39d9b3f28 2024-11-15T22:16:10,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/3cd033baf1e147c79fbd5086ff04cff9 is 50, key is test_row_0/C:col10/1731708968388/Put/seqid=0 2024-11-15T22:16:10,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741992_1168 (size=12151) 2024-11-15T22:16:10,795 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/3cd033baf1e147c79fbd5086ff04cff9 2024-11-15T22:16:10,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/8287d56eddcf46ee91b6d7263c171cca as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/8287d56eddcf46ee91b6d7263c171cca 2024-11-15T22:16:10,854 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/8287d56eddcf46ee91b6d7263c171cca, entries=150, sequenceid=194, filesize=30.4 K 2024-11-15T22:16:10,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/70eb828b120347da99bb15f39d9b3f28 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/70eb828b120347da99bb15f39d9b3f28 2024-11-15T22:16:10,891 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/70eb828b120347da99bb15f39d9b3f28, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T22:16:10,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/3cd033baf1e147c79fbd5086ff04cff9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/3cd033baf1e147c79fbd5086ff04cff9 2024-11-15T22:16:10,908 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/3cd033baf1e147c79fbd5086ff04cff9, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T22:16:10,916 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 7b87423906fae6354c3396098e18811c in 1312ms, sequenceid=194, compaction requested=false 2024-11-15T22:16:10,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:10,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:10,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-15T22:16:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-15T22:16:10,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-15T22:16:10,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1310 sec 2024-11-15T22:16:10,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.1440 sec 2024-11-15T22:16:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T22:16:10,931 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:10,932 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-15T22:16:10,934 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T22:16:10,937 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:10,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T22:16:11,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T22:16:11,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:11,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:11,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:11,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:11,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:11,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:11,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T22:16:11,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:11,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:11,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:11,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fdafec0c83054b588a3862fc1bb69d42_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708971090/Put/seqid=0 2024-11-15T22:16:11,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741993_1169 (size=14794) 2024-11-15T22:16:11,187 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:11,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709031223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:11,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:11,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:11,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:11,229 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fdafec0c83054b588a3862fc1bb69d42_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fdafec0c83054b588a3862fc1bb69d42_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:11,234 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/0077f5cf49cc47dfa9b9ac68c2b097f0, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:11,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/0077f5cf49cc47dfa9b9ac68c2b097f0 is 175, key is test_row_0/A:col10/1731708971090/Put/seqid=0 2024-11-15T22:16:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T22:16:11,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T22:16:11,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:11,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:11,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:11,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741994_1170 (size=39749) 2024-11-15T22:16:11,310 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/0077f5cf49cc47dfa9b9ac68c2b097f0 2024-11-15T22:16:11,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709031336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:11,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/b07f1956ae1d49e3b3f66c3fc9732bfb is 50, key is test_row_0/B:col10/1731708971090/Put/seqid=0 2024-11-15T22:16:11,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741995_1171 (size=12151) 2024-11-15T22:16:11,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T22:16:11,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:11,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:11,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:11,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709031552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:11,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:11,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:11,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T22:16:11,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T22:16:11,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:11,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:11,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:11,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T22:16:11,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:11,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:11,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:11,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/b07f1956ae1d49e3b3f66c3fc9732bfb 2024-11-15T22:16:11,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/e4dd1eb7c67640dfb447b42593de12db is 50, key is test_row_0/C:col10/1731708971090/Put/seqid=0 2024-11-15T22:16:11,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709031861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:11,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:11,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T22:16:11,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:11,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:11,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:11,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:11,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741996_1172 (size=12151) 2024-11-15T22:16:11,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/e4dd1eb7c67640dfb447b42593de12db 2024-11-15T22:16:11,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/0077f5cf49cc47dfa9b9ac68c2b097f0 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0077f5cf49cc47dfa9b9ac68c2b097f0 2024-11-15T22:16:12,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0077f5cf49cc47dfa9b9ac68c2b097f0, entries=200, sequenceid=216, filesize=38.8 K 2024-11-15T22:16:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/b07f1956ae1d49e3b3f66c3fc9732bfb as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b07f1956ae1d49e3b3f66c3fc9732bfb 2024-11-15T22:16:12,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b07f1956ae1d49e3b3f66c3fc9732bfb, entries=150, sequenceid=216, filesize=11.9 K 2024-11-15T22:16:12,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/e4dd1eb7c67640dfb447b42593de12db as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/e4dd1eb7c67640dfb447b42593de12db 2024-11-15T22:16:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T22:16:12,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T22:16:12,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:12,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:12,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:12,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:12,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/e4dd1eb7c67640dfb447b42593de12db, entries=150, sequenceid=216, filesize=11.9 K 2024-11-15T22:16:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,109 INFO [master/e611192d6313:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-15T22:16:12,109 INFO [master/e611192d6313:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-15T22:16:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7b87423906fae6354c3396098e18811c in 1022ms, sequenceid=216, compaction requested=true 2024-11-15T22:16:12,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:12,116 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:12,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:12,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:12,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:12,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:12,117 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:12,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:12,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,123 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,123 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/A is initiating minor compaction (all files) 2024-11-15T22:16:12,123 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/A in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:12,124 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ad186fbb88434bd39cb1009dd77c0601, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/8287d56eddcf46ee91b6d7263c171cca, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0077f5cf49cc47dfa9b9ac68c2b097f0] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=99.8 K 2024-11-15T22:16:12,124 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:12,124 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ad186fbb88434bd39cb1009dd77c0601, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/8287d56eddcf46ee91b6d7263c171cca, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0077f5cf49cc47dfa9b9ac68c2b097f0] 2024-11-15T22:16:12,124 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:12,125 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad186fbb88434bd39cb1009dd77c0601, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731708967170 2024-11-15T22:16:12,125 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/B is initiating minor compaction (all files) 2024-11-15T22:16:12,125 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/B in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:12,125 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/8bf0205f09d34ae59850d0a1050e9ce7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/70eb828b120347da99bb15f39d9b3f28, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b07f1956ae1d49e3b3f66c3fc9732bfb] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=35.8 K 2024-11-15T22:16:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,126 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bf0205f09d34ae59850d0a1050e9ce7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731708967170 2024-11-15T22:16:12,126 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8287d56eddcf46ee91b6d7263c171cca, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731708968388 2024-11-15T22:16:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,132 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 70eb828b120347da99bb15f39d9b3f28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731708968388 2024-11-15T22:16:12,132 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0077f5cf49cc47dfa9b9ac68c2b097f0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731708969746 2024-11-15T22:16:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,136 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting b07f1956ae1d49e3b3f66c3fc9732bfb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731708969813 2024-11-15T22:16:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,201 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,210 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#B#compaction#148 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:12,210 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/4acdb7413d3642fe913a8c067303dfc1 is 50, key is test_row_0/B:col10/1731708971090/Put/seqid=0 2024-11-15T22:16:12,213 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411159e99aacafeb34dbe89b6c7e181b225b6_7b87423906fae6354c3396098e18811c store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,217 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411159e99aacafeb34dbe89b6c7e181b225b6_7b87423906fae6354c3396098e18811c, store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:12,218 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159e99aacafeb34dbe89b6c7e181b225b6_7b87423906fae6354c3396098e18811c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T22:16:12,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:12,245 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T22:16:12,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:12,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:12,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:12,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:12,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:12,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741997_1173 (size=12493) 2024-11-15T22:16:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741998_1174 (size=4469) 2024-11-15T22:16:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159bbc280c719149f2ac062ab838103230_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708971190/Put/seqid=0 2024-11-15T22:16:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741999_1175 (size=9814) 2024-11-15T22:16:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709032659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:12,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:12,696 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#A#compaction#147 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:12,697 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/2bb476d9479d41c892d9528dbc2d8735 is 175, key is test_row_0/A:col10/1731708971090/Put/seqid=0 2024-11-15T22:16:12,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709032664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709032664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:12,725 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/4acdb7413d3642fe913a8c067303dfc1 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/4acdb7413d3642fe913a8c067303dfc1 2024-11-15T22:16:12,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:12,769 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159bbc280c719149f2ac062ab838103230_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159bbc280c719149f2ac062ab838103230_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:12,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742000_1176 (size=31447) 2024-11-15T22:16:12,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/7543b6c0ce7e4122b203cbe77ee287e5, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:12,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/7543b6c0ce7e4122b203cbe77ee287e5 is 175, key is test_row_0/A:col10/1731708971190/Put/seqid=0 2024-11-15T22:16:12,777 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b87423906fae6354c3396098e18811c/B of 7b87423906fae6354c3396098e18811c into 4acdb7413d3642fe913a8c067303dfc1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:12,777 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:12,777 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/B, priority=13, startTime=1731708972117; duration=0sec 2024-11-15T22:16:12,778 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:12,778 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:B 2024-11-15T22:16:12,778 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:12,787 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:12,787 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/C is initiating minor compaction (all files) 2024-11-15T22:16:12,787 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/C in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:12,787 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/4297f1cb14ba47f294c15facf141e7ac, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/3cd033baf1e147c79fbd5086ff04cff9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/e4dd1eb7c67640dfb447b42593de12db] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=35.8 K 2024-11-15T22:16:12,788 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 4297f1cb14ba47f294c15facf141e7ac, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731708967170 2024-11-15T22:16:12,788 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cd033baf1e147c79fbd5086ff04cff9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731708968388 2024-11-15T22:16:12,792 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting e4dd1eb7c67640dfb447b42593de12db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731708969813 2024-11-15T22:16:12,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709032792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:12,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:12,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:12,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:12,814 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/2bb476d9479d41c892d9528dbc2d8735 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/2bb476d9479d41c892d9528dbc2d8735 2024-11-15T22:16:12,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742001_1177 (size=22461) 2024-11-15T22:16:12,828 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/7543b6c0ce7e4122b203cbe77ee287e5 2024-11-15T22:16:12,832 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:12,833 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b87423906fae6354c3396098e18811c/A of 7b87423906fae6354c3396098e18811c into 2bb476d9479d41c892d9528dbc2d8735(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:12,833 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:12,833 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/A, priority=13, startTime=1731708972116; duration=0sec 2024-11-15T22:16:12,834 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:12,834 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:A 2024-11-15T22:16:12,838 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/940227fc62da4c74a1fd85ad27135cff is 50, key is test_row_0/C:col10/1731708971090/Put/seqid=0 2024-11-15T22:16:12,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709032815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:12,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709032833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:12,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/284d357a589747079d41a234b4ee31a2 is 50, key is test_row_0/B:col10/1731708971190/Put/seqid=0 2024-11-15T22:16:12,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742002_1178 (size=12493) 2024-11-15T22:16:12,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742003_1179 (size=9757) 2024-11-15T22:16:12,933 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/284d357a589747079d41a234b4ee31a2 2024-11-15T22:16:13,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709033075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709033075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709033075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/0e5b273218c94ffe95f9b3159fe7953f is 50, key is test_row_0/C:col10/1731708971190/Put/seqid=0 2024-11-15T22:16:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T22:16:13,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742004_1180 (size=9757) 2024-11-15T22:16:13,148 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/0e5b273218c94ffe95f9b3159fe7953f 2024-11-15T22:16:13,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/7543b6c0ce7e4122b203cbe77ee287e5 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/7543b6c0ce7e4122b203cbe77ee287e5 2024-11-15T22:16:13,201 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/7543b6c0ce7e4122b203cbe77ee287e5, entries=100, sequenceid=229, filesize=21.9 K 2024-11-15T22:16:13,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/284d357a589747079d41a234b4ee31a2 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/284d357a589747079d41a234b4ee31a2 2024-11-15T22:16:13,222 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/284d357a589747079d41a234b4ee31a2, entries=100, sequenceid=229, filesize=9.5 K 2024-11-15T22:16:13,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/0e5b273218c94ffe95f9b3159fe7953f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/0e5b273218c94ffe95f9b3159fe7953f 2024-11-15T22:16:13,240 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/0e5b273218c94ffe95f9b3159fe7953f, entries=100, sequenceid=229, filesize=9.5 K 2024-11-15T22:16:13,245 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 7b87423906fae6354c3396098e18811c in 1001ms, sequenceid=229, compaction requested=false 2024-11-15T22:16:13,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:13,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:13,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-15T22:16:13,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-15T22:16:13,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-15T22:16:13,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3140 sec 2024-11-15T22:16:13,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.3280 sec 2024-11-15T22:16:13,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708972633, max timestamp=1731708973394Num uniques 859; 2024-11-15T22:16:13,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:16:13,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-15T22:16:13,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:13,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:13,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:13,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:13,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:13,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:13,430 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/940227fc62da4c74a1fd85ad27135cff as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/940227fc62da4c74a1fd85ad27135cff 2024-11-15T22:16:13,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d8a6f1dc5e4147758174560fc7bf6151_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708972633/Put/seqid=0 2024-11-15T22:16:13,439 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b87423906fae6354c3396098e18811c/C of 7b87423906fae6354c3396098e18811c into 940227fc62da4c74a1fd85ad27135cff(size=12.2 K), total size for store is 21.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:13,439 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:13,439 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/C, priority=13, startTime=1731708972117; duration=0sec 2024-11-15T22:16:13,439 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:13,439 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:C 2024-11-15T22:16:13,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742005_1181 (size=14794) 2024-11-15T22:16:13,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709033472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709033475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709033481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709033481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709033491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709033603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709033612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709033612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709033621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709033612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709033824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709033833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709033843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709033854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:13,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709033865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:13,882 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:13,901 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d8a6f1dc5e4147758174560fc7bf6151_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d8a6f1dc5e4147758174560fc7bf6151_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:13,907 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/36ff8c2bbcd94853aedcfd581feed2b2, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:13,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/36ff8c2bbcd94853aedcfd581feed2b2 is 175, key is test_row_0/A:col10/1731708972633/Put/seqid=0 2024-11-15T22:16:13,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742006_1182 (size=39749) 2024-11-15T22:16:13,969 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/36ff8c2bbcd94853aedcfd581feed2b2 2024-11-15T22:16:14,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/ad469ce54bbc4eaa803c8236580bc101 is 50, key is test_row_0/B:col10/1731708972633/Put/seqid=0 2024-11-15T22:16:14,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742007_1183 (size=12151) 2024-11-15T22:16:14,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/ad469ce54bbc4eaa803c8236580bc101 2024-11-15T22:16:14,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/b40d94ca94ff47d080864dd15359b172 is 50, key is test_row_0/C:col10/1731708972633/Put/seqid=0 2024-11-15T22:16:14,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742008_1184 (size=12151) 2024-11-15T22:16:14,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/b40d94ca94ff47d080864dd15359b172 2024-11-15T22:16:14,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/36ff8c2bbcd94853aedcfd581feed2b2 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/36ff8c2bbcd94853aedcfd581feed2b2 2024-11-15T22:16:14,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/36ff8c2bbcd94853aedcfd581feed2b2, entries=200, sequenceid=252, filesize=38.8 K 2024-11-15T22:16:14,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/ad469ce54bbc4eaa803c8236580bc101 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/ad469ce54bbc4eaa803c8236580bc101 2024-11-15T22:16:14,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/ad469ce54bbc4eaa803c8236580bc101, entries=150, sequenceid=252, filesize=11.9 K 2024-11-15T22:16:14,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/b40d94ca94ff47d080864dd15359b172 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/b40d94ca94ff47d080864dd15359b172 2024-11-15T22:16:14,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/b40d94ca94ff47d080864dd15359b172, entries=150, sequenceid=252, filesize=11.9 K 2024-11-15T22:16:14,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 7b87423906fae6354c3396098e18811c in 762ms, sequenceid=252, compaction requested=true 2024-11-15T22:16:14,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:14,166 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:14,167 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:14,168 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/A is initiating minor compaction (all files) 2024-11-15T22:16:14,168 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/A in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:14,168 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/2bb476d9479d41c892d9528dbc2d8735, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/7543b6c0ce7e4122b203cbe77ee287e5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/36ff8c2bbcd94853aedcfd581feed2b2] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=91.5 K 2024-11-15T22:16:14,168 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:14,168 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/2bb476d9479d41c892d9528dbc2d8735, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/7543b6c0ce7e4122b203cbe77ee287e5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/36ff8c2bbcd94853aedcfd581feed2b2] 2024-11-15T22:16:14,169 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bb476d9479d41c892d9528dbc2d8735, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731708969813 2024-11-15T22:16:14,169 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7543b6c0ce7e4122b203cbe77ee287e5, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731708971190 2024-11-15T22:16:14,170 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36ff8c2bbcd94853aedcfd581feed2b2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731708972633 2024-11-15T22:16:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:14,181 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:14,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:14,183 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:14,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T22:16:14,183 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/B is initiating minor compaction (all files) 2024-11-15T22:16:14,183 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/B in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:14,183 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/4acdb7413d3642fe913a8c067303dfc1, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/284d357a589747079d41a234b4ee31a2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/ad469ce54bbc4eaa803c8236580bc101] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=33.6 K 2024-11-15T22:16:14,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:14,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:14,184 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 4acdb7413d3642fe913a8c067303dfc1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731708969813 2024-11-15T22:16:14,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:14,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:14,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:14,184 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 284d357a589747079d41a234b4ee31a2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731708971190 2024-11-15T22:16:14,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:14,185 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting ad469ce54bbc4eaa803c8236580bc101, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731708972633 2024-11-15T22:16:14,192 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:14,208 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#B#compaction#157 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:14,209 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/73ececb1305c4aa3b3939f93b3386561 is 50, key is test_row_0/B:col10/1731708972633/Put/seqid=0 2024-11-15T22:16:14,209 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115e38edf34e1d3486aada347cd80de1dc4_7b87423906fae6354c3396098e18811c store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:14,211 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115e38edf34e1d3486aada347cd80de1dc4_7b87423906fae6354c3396098e18811c, store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:14,211 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e38edf34e1d3486aada347cd80de1dc4_7b87423906fae6354c3396098e18811c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:14,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709034241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709034245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709034246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709034269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150cc4f066fe794575a75aba44a04e6ad0_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708974174/Put/seqid=0 2024-11-15T22:16:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709034274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742009_1185 (size=12595) 2024-11-15T22:16:14,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742010_1186 (size=4469) 2024-11-15T22:16:14,323 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#A#compaction#156 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:14,323 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/17382afc3bc742ba8e7d657216dffb40 is 175, key is test_row_0/A:col10/1731708972633/Put/seqid=0 2024-11-15T22:16:14,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742011_1187 (size=12454) 2024-11-15T22:16:14,356 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:14,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742012_1188 (size=31549) 2024-11-15T22:16:14,378 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150cc4f066fe794575a75aba44a04e6ad0_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150cc4f066fe794575a75aba44a04e6ad0_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:14,386 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/c080cfccdbee4a3b8cf5f7200fcd5d89, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:14,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/c080cfccdbee4a3b8cf5f7200fcd5d89 is 175, key is test_row_0/A:col10/1731708974174/Put/seqid=0 2024-11-15T22:16:14,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,389 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/17382afc3bc742ba8e7d657216dffb40 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/17382afc3bc742ba8e7d657216dffb40 2024-11-15T22:16:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709034371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709034382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709034382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709034384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709034401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,412 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b87423906fae6354c3396098e18811c/A of 7b87423906fae6354c3396098e18811c into 17382afc3bc742ba8e7d657216dffb40(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:14,412 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:14,412 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/A, priority=13, startTime=1731708974166; duration=0sec 2024-11-15T22:16:14,413 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:14,413 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:A 2024-11-15T22:16:14,413 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:14,417 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:14,417 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/C is initiating minor compaction (all files) 2024-11-15T22:16:14,417 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/C in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:14,417 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/940227fc62da4c74a1fd85ad27135cff, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/0e5b273218c94ffe95f9b3159fe7953f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/b40d94ca94ff47d080864dd15359b172] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=33.6 K 2024-11-15T22:16:14,419 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 940227fc62da4c74a1fd85ad27135cff, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731708969813 2024-11-15T22:16:14,419 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e5b273218c94ffe95f9b3159fe7953f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731708971190 2024-11-15T22:16:14,422 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40d94ca94ff47d080864dd15359b172, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731708972633 2024-11-15T22:16:14,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742013_1189 (size=31255) 2024-11-15T22:16:14,475 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#C#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:14,476 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/ddf4058cc9cc47deb62b49e73861e1bd is 50, key is test_row_0/C:col10/1731708972633/Put/seqid=0 2024-11-15T22:16:14,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742014_1190 (size=12595) 2024-11-15T22:16:14,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709034605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709034605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709034605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709034612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709034612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,714 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/73ececb1305c4aa3b3939f93b3386561 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/73ececb1305c4aa3b3939f93b3386561 2024-11-15T22:16:14,732 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b87423906fae6354c3396098e18811c/B of 7b87423906fae6354c3396098e18811c into 73ececb1305c4aa3b3939f93b3386561(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:14,732 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:14,733 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/B, priority=13, startTime=1731708974181; duration=0sec 2024-11-15T22:16:14,733 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:14,733 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:B 2024-11-15T22:16:14,849 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/c080cfccdbee4a3b8cf5f7200fcd5d89 2024-11-15T22:16:14,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/9a5f91106654441a91c94914ca2d1448 is 50, key is test_row_0/B:col10/1731708974174/Put/seqid=0 2024-11-15T22:16:14,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742015_1191 (size=12301) 2024-11-15T22:16:14,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709034937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709034937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/9a5f91106654441a91c94914ca2d1448 2024-11-15T22:16:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709034937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709034937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709034938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:14,996 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/ddf4058cc9cc47deb62b49e73861e1bd as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/ddf4058cc9cc47deb62b49e73861e1bd 2024-11-15T22:16:15,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/c02b7b4e94794fe5bcd400a7e86ee879 is 50, key is test_row_0/C:col10/1731708974174/Put/seqid=0 2024-11-15T22:16:15,041 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b87423906fae6354c3396098e18811c/C of 7b87423906fae6354c3396098e18811c into ddf4058cc9cc47deb62b49e73861e1bd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:15,041 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:15,041 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/C, priority=13, startTime=1731708974181; duration=0sec 2024-11-15T22:16:15,042 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:15,042 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:C 2024-11-15T22:16:15,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742016_1192 (size=12301) 2024-11-15T22:16:15,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/c02b7b4e94794fe5bcd400a7e86ee879 2024-11-15T22:16:15,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/c080cfccdbee4a3b8cf5f7200fcd5d89 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c080cfccdbee4a3b8cf5f7200fcd5d89 2024-11-15T22:16:15,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c080cfccdbee4a3b8cf5f7200fcd5d89, entries=150, sequenceid=270, filesize=30.5 K 2024-11-15T22:16:15,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/9a5f91106654441a91c94914ca2d1448 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/9a5f91106654441a91c94914ca2d1448 2024-11-15T22:16:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T22:16:15,131 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:15,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/9a5f91106654441a91c94914ca2d1448, entries=150, sequenceid=270, filesize=12.0 K 2024-11-15T22:16:15,137 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:15,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/c02b7b4e94794fe5bcd400a7e86ee879 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/c02b7b4e94794fe5bcd400a7e86ee879 2024-11-15T22:16:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-15T22:16:15,149 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:15,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/c02b7b4e94794fe5bcd400a7e86ee879, entries=150, sequenceid=270, filesize=12.0 K 2024-11-15T22:16:15,151 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:15,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T22:16:15,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 7b87423906fae6354c3396098e18811c in 977ms, sequenceid=270, compaction requested=false 2024-11-15T22:16:15,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T22:16:15,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T22:16:15,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:15,310 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T22:16:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:15,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f5c239b2ad3840ec91bb6e89a51b180e_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708974240/Put/seqid=0 2024-11-15T22:16:15,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742017_1193 (size=12454) 2024-11-15T22:16:15,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:15,429 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f5c239b2ad3840ec91bb6e89a51b180e_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f5c239b2ad3840ec91bb6e89a51b180e_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:15,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/b082f7270b494d808572c0f67741db67, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:15,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/b082f7270b494d808572c0f67741db67 is 175, key is test_row_0/A:col10/1731708974240/Put/seqid=0 2024-11-15T22:16:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T22:16:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:15,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742018_1194 (size=31255) 2024-11-15T22:16:15,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709035534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:15,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:15,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709035544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:15,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:15,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:15,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709035548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709035548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:15,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709035550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:15,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709035668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:15,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709035672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:15,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709035672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:15,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709035672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:15,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709035681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:15,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T22:16:15,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709035888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:15,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709035891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:15,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709035891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:15,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709035891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:15,906 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/b082f7270b494d808572c0f67741db67 2024-11-15T22:16:15,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709035905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:15,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/31811d9fc0c44f2e89430cbf2492d39e is 50, key is test_row_0/B:col10/1731708974240/Put/seqid=0 2024-11-15T22:16:16,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742019_1195 (size=12301) 2024-11-15T22:16:16,026 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/31811d9fc0c44f2e89430cbf2492d39e 2024-11-15T22:16:16,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/d9402eefa4874f6e880738def8bbecbf is 50, key is test_row_0/C:col10/1731708974240/Put/seqid=0 2024-11-15T22:16:16,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742020_1196 (size=12301) 2024-11-15T22:16:16,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709036218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:16,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709036220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:16,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:16,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:16,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709036220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:16,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:16,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709036220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709036222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:16,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T22:16:16,556 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/d9402eefa4874f6e880738def8bbecbf 2024-11-15T22:16:16,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/b082f7270b494d808572c0f67741db67 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/b082f7270b494d808572c0f67741db67 2024-11-15T22:16:16,601 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/b082f7270b494d808572c0f67741db67, entries=150, sequenceid=289, filesize=30.5 K 2024-11-15T22:16:16,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/31811d9fc0c44f2e89430cbf2492d39e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/31811d9fc0c44f2e89430cbf2492d39e 2024-11-15T22:16:16,623 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/31811d9fc0c44f2e89430cbf2492d39e, entries=150, sequenceid=289, filesize=12.0 K 2024-11-15T22:16:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/d9402eefa4874f6e880738def8bbecbf as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d9402eefa4874f6e880738def8bbecbf 2024-11-15T22:16:16,658 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d9402eefa4874f6e880738def8bbecbf, entries=150, sequenceid=289, filesize=12.0 K 2024-11-15T22:16:16,660 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7b87423906fae6354c3396098e18811c in 1350ms, sequenceid=289, compaction requested=true 2024-11-15T22:16:16,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:16,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:16,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-15T22:16:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-15T22:16:16,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-15T22:16:16,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5140 sec 2024-11-15T22:16:16,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.5310 sec 2024-11-15T22:16:16,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T22:16:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:16,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:16,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:16,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:16,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:16,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:16,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:16,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c26a2626f37d445bb01af9bede7473b6_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708975523/Put/seqid=0 2024-11-15T22:16:16,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709036792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:16,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709036800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:16,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:16,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709036801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:16,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:16,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:16,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742021_1197 (size=14994) 2024-11-15T22:16:16,850 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:16,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709036841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:16,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709036849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:16,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:16,907 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c26a2626f37d445bb01af9bede7473b6_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c26a2626f37d445bb01af9bede7473b6_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:16,912 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/9df502db58cb4fd0be753876c974c924, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:16,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/9df502db58cb4fd0be753876c974c924 is 175, key is test_row_0/A:col10/1731708975523/Put/seqid=0 2024-11-15T22:16:16,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742022_1198 (size=39949) 2024-11-15T22:16:16,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709036956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:16,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709036956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709036956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:16,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709036980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:16,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709036980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:17,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709037173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:17,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709037183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:17,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709037183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:17,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709037203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:17,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:17,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709037211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:17,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T22:16:17,311 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:17,312 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-15T22:16:17,316 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:17,317 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:17,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T22:16:17,355 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/9df502db58cb4fd0be753876c974c924 2024-11-15T22:16:17,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/343896cde95e4d69815c73df76e6ceae is 50, key is test_row_0/B:col10/1731708975523/Put/seqid=0 2024-11-15T22:16:17,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T22:16:17,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742023_1199 (size=12301) 2024-11-15T22:16:17,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/343896cde95e4d69815c73df76e6ceae 2024-11-15T22:16:17,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T22:16:17,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:17,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:17,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:17,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:17,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:17,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:17,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:17,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709037486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:17,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709037502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:17,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709037502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:17,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:17,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709037524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:17,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:17,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709037532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:17,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/b908d3bf4d87486db3f322f423a166a7 is 50, key is test_row_0/C:col10/1731708975523/Put/seqid=0 2024-11-15T22:16:17,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742024_1200 (size=12301) 2024-11-15T22:16:17,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/b908d3bf4d87486db3f322f423a166a7 2024-11-15T22:16:17,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/9df502db58cb4fd0be753876c974c924 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/9df502db58cb4fd0be753876c974c924 2024-11-15T22:16:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T22:16:17,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/9df502db58cb4fd0be753876c974c924, entries=200, sequenceid=307, filesize=39.0 K 2024-11-15T22:16:17,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/343896cde95e4d69815c73df76e6ceae as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/343896cde95e4d69815c73df76e6ceae 2024-11-15T22:16:17,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T22:16:17,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:17,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:17,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:17,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:17,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:17,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/343896cde95e4d69815c73df76e6ceae, entries=150, sequenceid=307, filesize=12.0 K 2024-11-15T22:16:17,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/b908d3bf4d87486db3f322f423a166a7 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/b908d3bf4d87486db3f322f423a166a7 2024-11-15T22:16:17,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/b908d3bf4d87486db3f322f423a166a7, entries=150, sequenceid=307, filesize=12.0 K 2024-11-15T22:16:17,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 7b87423906fae6354c3396098e18811c in 928ms, sequenceid=307, compaction requested=true 2024-11-15T22:16:17,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:17,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:17,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:17,675 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:16:17,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:17,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:17,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:17,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:16:17,678 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:16:17,686 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:16:17,686 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/A is initiating minor compaction (all files) 2024-11-15T22:16:17,686 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/A in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:17,686 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/17382afc3bc742ba8e7d657216dffb40, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c080cfccdbee4a3b8cf5f7200fcd5d89, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/b082f7270b494d808572c0f67741db67, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/9df502db58cb4fd0be753876c974c924] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=130.9 K 2024-11-15T22:16:17,686 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:17,686 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/17382afc3bc742ba8e7d657216dffb40, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c080cfccdbee4a3b8cf5f7200fcd5d89, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/b082f7270b494d808572c0f67741db67, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/9df502db58cb4fd0be753876c974c924] 2024-11-15T22:16:17,687 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 17382afc3bc742ba8e7d657216dffb40, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731708972633 2024-11-15T22:16:17,687 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c080cfccdbee4a3b8cf5f7200fcd5d89, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731708973464 2024-11-15T22:16:17,689 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting b082f7270b494d808572c0f67741db67, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731708974240 2024-11-15T22:16:17,689 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 9df502db58cb4fd0be753876c974c924, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731708975523 2024-11-15T22:16:17,690 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:16:17,690 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/B is initiating minor compaction (all files) 2024-11-15T22:16:17,690 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/B in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:17,690 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/73ececb1305c4aa3b3939f93b3386561, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/9a5f91106654441a91c94914ca2d1448, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/31811d9fc0c44f2e89430cbf2492d39e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/343896cde95e4d69815c73df76e6ceae] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=48.3 K 2024-11-15T22:16:17,693 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73ececb1305c4aa3b3939f93b3386561, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731708972633 2024-11-15T22:16:17,699 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a5f91106654441a91c94914ca2d1448, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731708973464 2024-11-15T22:16:17,700 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31811d9fc0c44f2e89430cbf2492d39e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731708974240 2024-11-15T22:16:17,701 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 343896cde95e4d69815c73df76e6ceae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731708975523 2024-11-15T22:16:17,749 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:17,755 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#B#compaction#169 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:17,756 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/7bcd65f0ca8c4d67b167579df412e81b is 50, key is test_row_0/B:col10/1731708975523/Put/seqid=0 2024-11-15T22:16:17,773 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115e2f601e3b9d547b08cd5f8bae9233645_7b87423906fae6354c3396098e18811c store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:17,776 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115e2f601e3b9d547b08cd5f8bae9233645_7b87423906fae6354c3396098e18811c, store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:17,777 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e2f601e3b9d547b08cd5f8bae9233645_7b87423906fae6354c3396098e18811c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:17,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T22:16:17,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:17,798 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T22:16:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:17,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742025_1201 (size=12881) 2024-11-15T22:16:17,843 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/7bcd65f0ca8c4d67b167579df412e81b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/7bcd65f0ca8c4d67b167579df412e81b 2024-11-15T22:16:17,853 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7b87423906fae6354c3396098e18811c/B of 7b87423906fae6354c3396098e18811c into 7bcd65f0ca8c4d67b167579df412e81b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:17,854 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:17,854 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/B, priority=12, startTime=1731708977675; duration=0sec 2024-11-15T22:16:17,854 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:17,854 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:B 2024-11-15T22:16:17,854 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:16:17,857 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:16:17,857 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/C is initiating minor compaction (all files) 2024-11-15T22:16:17,857 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/C in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:17,857 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/ddf4058cc9cc47deb62b49e73861e1bd, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/c02b7b4e94794fe5bcd400a7e86ee879, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d9402eefa4874f6e880738def8bbecbf, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/b908d3bf4d87486db3f322f423a166a7] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=48.3 K 2024-11-15T22:16:17,858 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddf4058cc9cc47deb62b49e73861e1bd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731708972633 2024-11-15T22:16:17,859 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting c02b7b4e94794fe5bcd400a7e86ee879, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731708973464 2024-11-15T22:16:17,860 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9402eefa4874f6e880738def8bbecbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731708974240 2024-11-15T22:16:17,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742026_1202 (size=4469) 2024-11-15T22:16:17,861 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting b908d3bf4d87486db3f322f423a166a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731708975523 2024-11-15T22:16:17,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115feea48df6fac49d8a50972537ef8ed4b_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708976820/Put/seqid=0 2024-11-15T22:16:17,867 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#A#compaction#168 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:17,867 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/77aef2e4de32452abb273062d083269f is 175, key is test_row_0/A:col10/1731708975523/Put/seqid=0 2024-11-15T22:16:17,913 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#C#compaction#171 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:17,913 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/d6a6542c8b57485d937a222aa787f0f9 is 50, key is test_row_0/C:col10/1731708975523/Put/seqid=0 2024-11-15T22:16:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T22:16:17,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742027_1203 (size=31835) 2024-11-15T22:16:17,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742028_1204 (size=12454) 2024-11-15T22:16:17,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:17,991 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115feea48df6fac49d8a50972537ef8ed4b_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115feea48df6fac49d8a50972537ef8ed4b_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:17,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/46ff3571a6e34f50ba1bea27ca25716d, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:17,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/46ff3571a6e34f50ba1bea27ca25716d is 175, key is test_row_0/A:col10/1731708976820/Put/seqid=0 2024-11-15T22:16:18,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742029_1205 (size=12881) 2024-11-15T22:16:18,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:18,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742030_1206 (size=31255) 2024-11-15T22:16:18,076 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/46ff3571a6e34f50ba1bea27ca25716d 2024-11-15T22:16:18,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/b0130503e70e46a99a29d04a4fe9b2f7 is 50, key is test_row_0/B:col10/1731708976820/Put/seqid=0 2024-11-15T22:16:18,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709038128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709038128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709038135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742031_1207 (size=12301) 2024-11-15T22:16:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709038137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709038141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,158 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/b0130503e70e46a99a29d04a4fe9b2f7 2024-11-15T22:16:18,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/e548bcef7d524cb998b0aa850fbf35c2 is 50, key is test_row_0/C:col10/1731708976820/Put/seqid=0 2024-11-15T22:16:18,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742032_1208 (size=12301) 2024-11-15T22:16:18,221 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/e548bcef7d524cb998b0aa850fbf35c2 2024-11-15T22:16:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/46ff3571a6e34f50ba1bea27ca25716d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/46ff3571a6e34f50ba1bea27ca25716d 2024-11-15T22:16:18,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709038252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,257 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/46ff3571a6e34f50ba1bea27ca25716d, entries=150, sequenceid=321, filesize=30.5 K 2024-11-15T22:16:18,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/b0130503e70e46a99a29d04a4fe9b2f7 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b0130503e70e46a99a29d04a4fe9b2f7 2024-11-15T22:16:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,267 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b0130503e70e46a99a29d04a4fe9b2f7, entries=150, sequenceid=321, filesize=12.0 K 2024-11-15T22:16:18,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/e548bcef7d524cb998b0aa850fbf35c2 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/e548bcef7d524cb998b0aa850fbf35c2 2024-11-15T22:16:18,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709038257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,278 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/e548bcef7d524cb998b0aa850fbf35c2, entries=150, sequenceid=321, filesize=12.0 K 2024-11-15T22:16:18,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709038257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,281 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 7b87423906fae6354c3396098e18811c in 483ms, sequenceid=321, compaction requested=false 2024-11-15T22:16:18,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:18,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:18,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-15T22:16:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-15T22:16:18,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T22:16:18,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:18,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:18,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:18,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:18,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:18,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:18,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-15T22:16:18,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 970 msec 2024-11-15T22:16:18,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 980 msec 2024-11-15T22:16:18,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111556d3f3b4ff844233b9242a6334c61e9b_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708978122/Put/seqid=0 2024-11-15T22:16:18,370 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/77aef2e4de32452abb273062d083269f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/77aef2e4de32452abb273062d083269f 2024-11-15T22:16:18,390 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7b87423906fae6354c3396098e18811c/A of 7b87423906fae6354c3396098e18811c into 77aef2e4de32452abb273062d083269f(size=31.1 K), total size for store is 61.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:18,390 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:18,390 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/A, priority=12, startTime=1731708977675; duration=0sec 2024-11-15T22:16:18,391 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:18,391 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:A 2024-11-15T22:16:18,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709038391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709038406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742033_1209 (size=14994) 2024-11-15T22:16:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T22:16:18,453 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:18,453 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-15T22:16:18,456 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:18,457 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:18,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T22:16:18,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709038472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709038483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,495 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/d6a6542c8b57485d937a222aa787f0f9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d6a6542c8b57485d937a222aa787f0f9 2024-11-15T22:16:18,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709038483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709038512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709038524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,557 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7b87423906fae6354c3396098e18811c/C of 7b87423906fae6354c3396098e18811c into d6a6542c8b57485d937a222aa787f0f9(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:18,557 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:18,557 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/C, priority=12, startTime=1731708977675; duration=0sec 2024-11-15T22:16:18,557 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:18,557 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:C 2024-11-15T22:16:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T22:16:18,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T22:16:18,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:18,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:18,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709038732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709038741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T22:16:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T22:16:18,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:18,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:18,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:18,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:18,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:18,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709038796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709038802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709038804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:18,822 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:18,846 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111556d3f3b4ff844233b9242a6334c61e9b_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111556d3f3b4ff844233b9242a6334c61e9b_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:18,853 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/91b19f878b5641b4a8abd615a6a34e23, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:18,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/91b19f878b5641b4a8abd615a6a34e23 is 175, key is test_row_0/A:col10/1731708978122/Put/seqid=0 2024-11-15T22:16:18,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742034_1210 (size=39949) 2024-11-15T22:16:18,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T22:16:18,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:18,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:19,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709039045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:19,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709039052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T22:16:19,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T22:16:19,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:19,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:19,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:19,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:19,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:19,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T22:16:19,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:19,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:19,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:19,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:19,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:19,297 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=341, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/91b19f878b5641b4a8abd615a6a34e23 2024-11-15T22:16:19,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709039312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:19,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709039322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:19,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709039322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:19,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/b767923cde46480c8d439409d13a2027 is 50, key is test_row_0/B:col10/1731708978122/Put/seqid=0 2024-11-15T22:16:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:19,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742035_1211 (size=12301) 2024-11-15T22:16:19,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/b767923cde46480c8d439409d13a2027 2024-11-15T22:16:19,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/6a2d05f3c8024a52bf47aecf4b5fe4a3 is 50, key is test_row_0/C:col10/1731708978122/Put/seqid=0 2024-11-15T22:16:19,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T22:16:19,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:19,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:19,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:19,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:19,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:19,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742036_1212 (size=12301) 2024-11-15T22:16:19,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/6a2d05f3c8024a52bf47aecf4b5fe4a3 2024-11-15T22:16:19,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/91b19f878b5641b4a8abd615a6a34e23 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/91b19f878b5641b4a8abd615a6a34e23 2024-11-15T22:16:19,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/91b19f878b5641b4a8abd615a6a34e23, entries=200, sequenceid=341, filesize=39.0 K 2024-11-15T22:16:19,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/b767923cde46480c8d439409d13a2027 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b767923cde46480c8d439409d13a2027 2024-11-15T22:16:19,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T22:16:19,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:19,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:19,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:19,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:19,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:19,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b767923cde46480c8d439409d13a2027, entries=150, sequenceid=341, filesize=12.0 K 2024-11-15T22:16:19,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/6a2d05f3c8024a52bf47aecf4b5fe4a3 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/6a2d05f3c8024a52bf47aecf4b5fe4a3 2024-11-15T22:16:19,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709039561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:19,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709039561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:19,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/6a2d05f3c8024a52bf47aecf4b5fe4a3, entries=150, sequenceid=341, filesize=12.0 K 2024-11-15T22:16:19,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7b87423906fae6354c3396098e18811c in 1305ms, sequenceid=341, compaction requested=true 2024-11-15T22:16:19,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:19,593 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:19,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:19,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:19,593 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:19,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:19,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:19,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:19,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:19,603 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:19,603 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/B is initiating minor compaction (all files) 2024-11-15T22:16:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T22:16:19,603 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/B in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:19,603 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/7bcd65f0ca8c4d67b167579df412e81b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b0130503e70e46a99a29d04a4fe9b2f7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b767923cde46480c8d439409d13a2027] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=36.6 K 2024-11-15T22:16:19,603 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:19,603 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/A is initiating minor compaction (all files) 2024-11-15T22:16:19,603 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/A in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:19,604 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/77aef2e4de32452abb273062d083269f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/46ff3571a6e34f50ba1bea27ca25716d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/91b19f878b5641b4a8abd615a6a34e23] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=100.6 K 2024-11-15T22:16:19,604 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:19,604 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/77aef2e4de32452abb273062d083269f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/46ff3571a6e34f50ba1bea27ca25716d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/91b19f878b5641b4a8abd615a6a34e23] 2024-11-15T22:16:19,605 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77aef2e4de32452abb273062d083269f, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731708975523 2024-11-15T22:16:19,605 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bcd65f0ca8c4d67b167579df412e81b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731708975523 2024-11-15T22:16:19,606 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46ff3571a6e34f50ba1bea27ca25716d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731708976793 2024-11-15T22:16:19,606 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting b0130503e70e46a99a29d04a4fe9b2f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731708976793 2024-11-15T22:16:19,608 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91b19f878b5641b4a8abd615a6a34e23, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731708978122 2024-11-15T22:16:19,608 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting b767923cde46480c8d439409d13a2027, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731708978122 2024-11-15T22:16:19,658 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:19,659 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#B#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:19,659 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/c89aa883e5f0487b9a366931b0d5a92f is 50, key is test_row_0/B:col10/1731708978122/Put/seqid=0 2024-11-15T22:16:19,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742037_1213 (size=12983) 2024-11-15T22:16:19,700 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115b0fd40ae2b674a5aa1ddb8745ccfc780_7b87423906fae6354c3396098e18811c store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:19,702 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115b0fd40ae2b674a5aa1ddb8745ccfc780_7b87423906fae6354c3396098e18811c, store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:19,703 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b0fd40ae2b674a5aa1ddb8745ccfc780_7b87423906fae6354c3396098e18811c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:19,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T22:16:19,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:19,716 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T22:16:19,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:19,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:19,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:19,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:19,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:19,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:19,719 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/c89aa883e5f0487b9a366931b0d5a92f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/c89aa883e5f0487b9a366931b0d5a92f 2024-11-15T22:16:19,732 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b87423906fae6354c3396098e18811c/B of 7b87423906fae6354c3396098e18811c into c89aa883e5f0487b9a366931b0d5a92f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:19,732 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:19,732 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/B, priority=13, startTime=1731708979593; duration=0sec 2024-11-15T22:16:19,732 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:19,732 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:B 2024-11-15T22:16:19,738 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:19,739 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:19,739 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/C is initiating minor compaction (all files) 2024-11-15T22:16:19,739 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/C in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:19,740 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d6a6542c8b57485d937a222aa787f0f9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/e548bcef7d524cb998b0aa850fbf35c2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/6a2d05f3c8024a52bf47aecf4b5fe4a3] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=36.6 K 2024-11-15T22:16:19,741 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d6a6542c8b57485d937a222aa787f0f9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731708975523 2024-11-15T22:16:19,741 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting e548bcef7d524cb998b0aa850fbf35c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731708976793 2024-11-15T22:16:19,741 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a2d05f3c8024a52bf47aecf4b5fe4a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731708978122 2024-11-15T22:16:19,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111588e9b020bc374a6e8ec045ac0df6a488_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708978402/Put/seqid=0 2024-11-15T22:16:19,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742038_1214 (size=4469) 2024-11-15T22:16:19,769 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:19,770 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/d2ba7e814e474f228454efa1ee73e2ec is 50, key is test_row_0/C:col10/1731708978122/Put/seqid=0 2024-11-15T22:16:19,777 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#A#compaction#178 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:19,778 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/f855207503cd4732a7806eb3bfc1a809 is 175, key is test_row_0/A:col10/1731708978122/Put/seqid=0 2024-11-15T22:16:19,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742039_1215 (size=12454) 2024-11-15T22:16:19,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742041_1217 (size=12983) 2024-11-15T22:16:19,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742040_1216 (size=31937) 2024-11-15T22:16:19,852 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/d2ba7e814e474f228454efa1ee73e2ec as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d2ba7e814e474f228454efa1ee73e2ec 2024-11-15T22:16:19,867 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b87423906fae6354c3396098e18811c/C of 7b87423906fae6354c3396098e18811c into d2ba7e814e474f228454efa1ee73e2ec(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:19,867 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:19,867 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/C, priority=13, startTime=1731708979593; duration=0sec 2024-11-15T22:16:19,867 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:19,867 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:C 2024-11-15T22:16:20,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:20,239 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111588e9b020bc374a6e8ec045ac0df6a488_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111588e9b020bc374a6e8ec045ac0df6a488_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:20,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/8f0740ece25e4b46b9aef2035656e419, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:20,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/8f0740ece25e4b46b9aef2035656e419 is 175, key is test_row_0/A:col10/1731708978402/Put/seqid=0 2024-11-15T22:16:20,256 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/f855207503cd4732a7806eb3bfc1a809 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/f855207503cd4732a7806eb3bfc1a809 2024-11-15T22:16:20,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742042_1218 (size=31255) 2024-11-15T22:16:20,268 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/8f0740ece25e4b46b9aef2035656e419 2024-11-15T22:16:20,271 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b87423906fae6354c3396098e18811c/A of 7b87423906fae6354c3396098e18811c into f855207503cd4732a7806eb3bfc1a809(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:20,271 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:20,271 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/A, priority=13, startTime=1731708979593; duration=0sec 2024-11-15T22:16:20,271 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:20,271 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:A 2024-11-15T22:16:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/4c130364bd934cfca19fae417e19c979 is 50, key is test_row_0/B:col10/1731708978402/Put/seqid=0 2024-11-15T22:16:20,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742043_1219 (size=12301) 2024-11-15T22:16:20,318 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/4c130364bd934cfca19fae417e19c979 2024-11-15T22:16:20,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/2e5284d314fb4618a2f18ad146a473cb is 50, key is test_row_0/C:col10/1731708978402/Put/seqid=0 2024-11-15T22:16:20,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742044_1220 (size=12301) 2024-11-15T22:16:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:20,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. as already flushing 2024-11-15T22:16:20,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709040385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:20,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709040388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:20,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709040395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:20,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709040502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:20,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709040502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709040502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:20,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53346 deadline: 1731709040582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:20,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53284 deadline: 1731709040586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:20,587 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_0' in 7b87423906fae6354c3396098e18811c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:16:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:20,588 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in 7b87423906fae6354c3396098e18811c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:16:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T22:16:20,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709040713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:20,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709040713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709040714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:20,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:20,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:20,749 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/2e5284d314fb4618a2f18ad146a473cb 2024-11-15T22:16:20,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/8f0740ece25e4b46b9aef2035656e419 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/8f0740ece25e4b46b9aef2035656e419 2024-11-15T22:16:20,796 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/8f0740ece25e4b46b9aef2035656e419, entries=150, sequenceid=356, filesize=30.5 K 2024-11-15T22:16:20,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/4c130364bd934cfca19fae417e19c979 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/4c130364bd934cfca19fae417e19c979 2024-11-15T22:16:20,811 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/4c130364bd934cfca19fae417e19c979, entries=150, sequenceid=356, filesize=12.0 K 2024-11-15T22:16:20,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/2e5284d314fb4618a2f18ad146a473cb as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/2e5284d314fb4618a2f18ad146a473cb 2024-11-15T22:16:20,830 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/2e5284d314fb4618a2f18ad146a473cb, entries=150, sequenceid=356, filesize=12.0 K 2024-11-15T22:16:20,837 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7b87423906fae6354c3396098e18811c in 1121ms, sequenceid=356, compaction requested=false 2024-11-15T22:16:20,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:20,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:20,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-15T22:16:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-15T22:16:20,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-15T22:16:20,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3910 sec 2024-11-15T22:16:20,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.4020 sec 2024-11-15T22:16:21,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T22:16:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:21,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731708980383, max timestamp=1731708981028Num uniques 859; 2024-11-15T22:16:21,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:16:21,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731708980383, max timestamp=1731708981028Num uniques 859; 2024-11-15T22:16:21,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:16:21,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:21,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:21,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:21,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:21,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:21,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:21,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cbf7f60152d44f03962f718feb906679_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708981026/Put/seqid=0 2024-11-15T22:16:21,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742045_1221 (size=14994) 2024-11-15T22:16:21,092 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:21,125 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cbf7f60152d44f03962f718feb906679_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cbf7f60152d44f03962f718feb906679_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:21,131 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/0df882ccf89c47baa9192b9334f36924, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:21,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/0df882ccf89c47baa9192b9334f36924 is 175, key is test_row_0/A:col10/1731708981026/Put/seqid=0 2024-11-15T22:16:21,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742046_1222 (size=39949) 2024-11-15T22:16:21,157 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/0df882ccf89c47baa9192b9334f36924 2024-11-15T22:16:21,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709041173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:21,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/db51e5575c4245b399833e92d465ac07 is 50, key is test_row_0/B:col10/1731708981026/Put/seqid=0 2024-11-15T22:16:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:21,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709041182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:21,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709041196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:21,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742047_1223 (size=12301) 2024-11-15T22:16:21,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709041302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:21,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:21,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709041312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:21,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:21,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709041316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:21,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:21,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709041512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:21,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709041531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:21,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709041531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:21,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/db51e5575c4245b399833e92d465ac07 2024-11-15T22:16:21,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/c2390ce3786c4e1a8fe6dbcbf73e9828 is 50, key is test_row_0/C:col10/1731708981026/Put/seqid=0 2024-11-15T22:16:21,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742048_1224 (size=12301) 2024-11-15T22:16:21,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53332 deadline: 1731709041822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:21,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1731709041844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53342 deadline: 1731709041844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:22,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/c2390ce3786c4e1a8fe6dbcbf73e9828 2024-11-15T22:16:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/0df882ccf89c47baa9192b9334f36924 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0df882ccf89c47baa9192b9334f36924 2024-11-15T22:16:22,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0df882ccf89c47baa9192b9334f36924, entries=200, sequenceid=380, filesize=39.0 K 2024-11-15T22:16:22,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/db51e5575c4245b399833e92d465ac07 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/db51e5575c4245b399833e92d465ac07 2024-11-15T22:16:22,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/db51e5575c4245b399833e92d465ac07, entries=150, sequenceid=380, filesize=12.0 K 2024-11-15T22:16:22,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/c2390ce3786c4e1a8fe6dbcbf73e9828 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/c2390ce3786c4e1a8fe6dbcbf73e9828 2024-11-15T22:16:22,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/c2390ce3786c4e1a8fe6dbcbf73e9828, entries=150, sequenceid=380, filesize=12.0 K 2024-11-15T22:16:22,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~62.62 KB/64120, heapSize ~174.28 KB/178464, currentSize=114.05 KB/116790 for 7b87423906fae6354c3396098e18811c in 1206ms, sequenceid=380, compaction requested=true 2024-11-15T22:16:22,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:22,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:22,235 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:22,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:22,235 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:22,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:22,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:22,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7b87423906fae6354c3396098e18811c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:22,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:22,237 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:22,237 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:22,237 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/A is initiating minor compaction (all files) 2024-11-15T22:16:22,237 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/B is initiating minor compaction (all files) 2024-11-15T22:16:22,237 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/A in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:22,237 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/B in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:22,237 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/f855207503cd4732a7806eb3bfc1a809, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/8f0740ece25e4b46b9aef2035656e419, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0df882ccf89c47baa9192b9334f36924] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=100.7 K 2024-11-15T22:16:22,237 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/c89aa883e5f0487b9a366931b0d5a92f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/4c130364bd934cfca19fae417e19c979, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/db51e5575c4245b399833e92d465ac07] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=36.7 K 2024-11-15T22:16:22,237 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:22,237 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/f855207503cd4732a7806eb3bfc1a809, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/8f0740ece25e4b46b9aef2035656e419, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0df882ccf89c47baa9192b9334f36924] 2024-11-15T22:16:22,238 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c89aa883e5f0487b9a366931b0d5a92f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731708978122 2024-11-15T22:16:22,238 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting f855207503cd4732a7806eb3bfc1a809, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731708978122 2024-11-15T22:16:22,238 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c130364bd934cfca19fae417e19c979, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731708978301 2024-11-15T22:16:22,238 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f0740ece25e4b46b9aef2035656e419, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731708978301 2024-11-15T22:16:22,239 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting db51e5575c4245b399833e92d465ac07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731708980392 2024-11-15T22:16:22,240 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0df882ccf89c47baa9192b9334f36924, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731708980383 2024-11-15T22:16:22,271 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#B#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:22,277 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/54ad03e8dd6d4e6e94ca60fd70bb2f5e is 50, key is test_row_0/B:col10/1731708981026/Put/seqid=0 2024-11-15T22:16:22,280 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:22,301 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111500fb20d7067649018ea94536c1010e62_7b87423906fae6354c3396098e18811c store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:22,303 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111500fb20d7067649018ea94536c1010e62_7b87423906fae6354c3396098e18811c, store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:22,303 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111500fb20d7067649018ea94536c1010e62_7b87423906fae6354c3396098e18811c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:22,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742049_1225 (size=13085) 2024-11-15T22:16:22,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708981103, max timestamp=1731708982344Num uniques 859; 2024-11-15T22:16:22,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:16:22,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708981103, max timestamp=1731708982344Num uniques 859; 2024-11-15T22:16:22,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:16:22,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708981103, max timestamp=1731708982344Num uniques 859; 2024-11-15T22:16:22,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:16:22,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742050_1226 (size=4469) 2024-11-15T22:16:22,365 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#A#compaction#187 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:22,366 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/7db1186f856341acaa462baafb1abea6 is 175, key is test_row_0/A:col10/1731708981026/Put/seqid=0 2024-11-15T22:16:22,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742051_1227 (size=32039) 2024-11-15T22:16:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:22,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=60.38 KB heapSize=165.84 KB 2024-11-15T22:16:22,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:22,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:22,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:22,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:22,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:22,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:22,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156d96790105fa4751a0c81efc0092b6ef_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708981103/Put/seqid=0 2024-11-15T22:16:22,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742052_1228 (size=20074) 2024-11-15T22:16:22,480 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:22,497 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-11-15T22:16:22,497 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:22,497 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,497 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,497 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:22,499 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-11-15T22:16:22,499 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:22,499 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,499 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:22,502 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156d96790105fa4751a0c81efc0092b6ef_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156d96790105fa4751a0c81efc0092b6ef_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:22,503 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-15T22:16:22,503 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:22,504 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,504 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,504 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:22,504 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/0d13b808f29f4b56ac286e77cf26f11e, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:22,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/0d13b808f29f4b56ac286e77cf26f11e is 175, key is test_row_0/A:col10/1731708981103/Put/seqid=0 2024-11-15T22:16:22,507 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-15T22:16:22,507 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:22,507 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,507 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,508 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:22,511 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-11-15T22:16:22,511 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:22,511 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,511 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:22,514 INFO [Thread-669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-669. 2024-11-15T22:16:22,515 DEBUG [Thread-669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:22,515 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,515 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:22,516 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-15T22:16:22,516 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:22,516 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,516 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:22,516 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742053_1229 (size=57333) 2024-11-15T22:16:22,617 INFO [Thread-666 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-666. 2024-11-15T22:16:22,617 DEBUG [Thread-666 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:22,617 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,617 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,617 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:22,618 INFO [Thread-664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-664. 2024-11-15T22:16:22,619 DEBUG [Thread-664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:22,619 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,619 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T22:16:22,652 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:22,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T22:16:22,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-15T22:16:22,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-15T22:16:22,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-15T22:16:22,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-15T22:16:22,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-15T22:16:22,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T22:16:22,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1503 2024-11-15T22:16:22,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1462 2024-11-15T22:16:22,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T22:16:22,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 713 2024-11-15T22:16:22,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2124 rows 2024-11-15T22:16:22,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 703 2024-11-15T22:16:22,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2103 rows 2024-11-15T22:16:22,652 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:16:22,652 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:16:22,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:22,653 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:22,654 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T22:16:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T22:16:22,667 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708982667"}]},"ts":"1731708982667"} 2024-11-15T22:16:22,672 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T22:16:22,672 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T22:16:22,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:16:22,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7b87423906fae6354c3396098e18811c, UNASSIGN}] 2024-11-15T22:16:22,679 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7b87423906fae6354c3396098e18811c, UNASSIGN 2024-11-15T22:16:22,681 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=7b87423906fae6354c3396098e18811c, regionState=CLOSING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:16:22,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7b87423906fae6354c3396098e18811c, UNASSIGN because future has completed 2024-11-15T22:16:22,684 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:16:22,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:16:22,717 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/54ad03e8dd6d4e6e94ca60fd70bb2f5e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/54ad03e8dd6d4e6e94ca60fd70bb2f5e 2024-11-15T22:16:22,727 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b87423906fae6354c3396098e18811c/B of 7b87423906fae6354c3396098e18811c into 54ad03e8dd6d4e6e94ca60fd70bb2f5e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:22,727 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:22,727 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/B, priority=13, startTime=1731708982235; duration=0sec 2024-11-15T22:16:22,728 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:22,728 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:B 2024-11-15T22:16:22,728 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:22,730 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:22,730 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 7b87423906fae6354c3396098e18811c/C is initiating minor compaction (all files) 2024-11-15T22:16:22,730 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7b87423906fae6354c3396098e18811c/C in TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:22,730 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d2ba7e814e474f228454efa1ee73e2ec, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/2e5284d314fb4618a2f18ad146a473cb, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/c2390ce3786c4e1a8fe6dbcbf73e9828] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp, totalSize=36.7 K 2024-11-15T22:16:22,731 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d2ba7e814e474f228454efa1ee73e2ec, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731708978122 2024-11-15T22:16:22,736 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e5284d314fb4618a2f18ad146a473cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731708978301 2024-11-15T22:16:22,739 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c2390ce3786c4e1a8fe6dbcbf73e9828, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731708980392 2024-11-15T22:16:22,758 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7b87423906fae6354c3396098e18811c#C#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:22,759 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/40ab1e9026d74da8a509d33390003544 is 50, key is test_row_0/C:col10/1731708981026/Put/seqid=0 2024-11-15T22:16:22,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742054_1230 (size=13085) 2024-11-15T22:16:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T22:16:22,775 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/40ab1e9026d74da8a509d33390003544 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/40ab1e9026d74da8a509d33390003544 2024-11-15T22:16:22,782 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b87423906fae6354c3396098e18811c/C of 7b87423906fae6354c3396098e18811c into 40ab1e9026d74da8a509d33390003544(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:22,782 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:22,782 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/C, priority=13, startTime=1731708982236; duration=0sec 2024-11-15T22:16:22,783 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:22,783 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:C 2024-11-15T22:16:22,788 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/7db1186f856341acaa462baafb1abea6 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/7db1186f856341acaa462baafb1abea6 2024-11-15T22:16:22,795 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b87423906fae6354c3396098e18811c/A of 7b87423906fae6354c3396098e18811c into 7db1186f856341acaa462baafb1abea6(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:22,795 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:22,795 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., storeName=7b87423906fae6354c3396098e18811c/A, priority=13, startTime=1731708982235; duration=0sec 2024-11-15T22:16:22,795 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:22,795 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7b87423906fae6354c3396098e18811c:A 2024-11-15T22:16:22,840 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:22,840 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:16:22,840 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing 7b87423906fae6354c3396098e18811c, disabling compactions & flushes 2024-11-15T22:16:22,840 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:22,925 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=408, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/0d13b808f29f4b56ac286e77cf26f11e 2024-11-15T22:16:22,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/ba30b813023e46cb88dfa6d00434e8ba is 50, key is test_row_0/B:col10/1731708981103/Put/seqid=0 2024-11-15T22:16:22,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742055_1231 (size=12301) 2024-11-15T22:16:22,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/ba30b813023e46cb88dfa6d00434e8ba 2024-11-15T22:16:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T22:16:22,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/846ffb6e76c44545a9c1a10aeb4afcdd is 50, key is test_row_0/C:col10/1731708981103/Put/seqid=0 2024-11-15T22:16:23,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742056_1232 (size=12301) 2024-11-15T22:16:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T22:16:23,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/846ffb6e76c44545a9c1a10aeb4afcdd 2024-11-15T22:16:23,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/0d13b808f29f4b56ac286e77cf26f11e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0d13b808f29f4b56ac286e77cf26f11e 2024-11-15T22:16:23,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0d13b808f29f4b56ac286e77cf26f11e, entries=300, sequenceid=408, filesize=56.0 K 2024-11-15T22:16:23,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/ba30b813023e46cb88dfa6d00434e8ba as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/ba30b813023e46cb88dfa6d00434e8ba 2024-11-15T22:16:23,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/ba30b813023e46cb88dfa6d00434e8ba, entries=150, sequenceid=408, filesize=12.0 K 2024-11-15T22:16:23,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/846ffb6e76c44545a9c1a10aeb4afcdd as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/846ffb6e76c44545a9c1a10aeb4afcdd 2024-11-15T22:16:23,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/846ffb6e76c44545a9c1a10aeb4afcdd, entries=150, sequenceid=408, filesize=12.0 K 2024-11-15T22:16:23,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~205.64 KB/210576, currentSize=93.93 KB/96180 for 7b87423906fae6354c3396098e18811c in 1143ms, sequenceid=408, compaction requested=false 2024-11-15T22:16:23,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7b87423906fae6354c3396098e18811c: 2024-11-15T22:16:23,544 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:23,544 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:23,544 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. after waiting 0 ms 2024-11-15T22:16:23,544 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:23,545 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing 7b87423906fae6354c3396098e18811c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T22:16:23,545 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=A 2024-11-15T22:16:23,545 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:23,545 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=B 2024-11-15T22:16:23,545 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:23,545 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7b87423906fae6354c3396098e18811c, store=C 2024-11-15T22:16:23,545 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:23,558 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115353f31f43e244a7da986ef0d11f6d064_7b87423906fae6354c3396098e18811c is 50, key is test_row_0/A:col10/1731708982604/Put/seqid=0 2024-11-15T22:16:23,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742057_1233 (size=12454) 2024-11-15T22:16:23,594 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:23,611 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115353f31f43e244a7da986ef0d11f6d064_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115353f31f43e244a7da986ef0d11f6d064_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:23,622 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/a9862700a40f48dfa39b42e8f4d6923f, store: [table=TestAcidGuarantees family=A region=7b87423906fae6354c3396098e18811c] 2024-11-15T22:16:23,623 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/a9862700a40f48dfa39b42e8f4d6923f is 175, key is test_row_0/A:col10/1731708982604/Put/seqid=0 2024-11-15T22:16:23,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742058_1234 (size=31255) 2024-11-15T22:16:23,650 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=428, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/a9862700a40f48dfa39b42e8f4d6923f 2024-11-15T22:16:23,690 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/5c579e7d2e164c9e8b7b81ecf9f4803e is 50, key is test_row_0/B:col10/1731708982604/Put/seqid=0 2024-11-15T22:16:23,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742059_1235 (size=12301) 2024-11-15T22:16:23,699 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/5c579e7d2e164c9e8b7b81ecf9f4803e 2024-11-15T22:16:23,713 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/f4f07fe3dde04f2fba16021cb97134ac is 50, key is test_row_0/C:col10/1731708982604/Put/seqid=0 2024-11-15T22:16:23,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742060_1236 (size=12301) 2024-11-15T22:16:23,724 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/f4f07fe3dde04f2fba16021cb97134ac 2024-11-15T22:16:23,745 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/A/a9862700a40f48dfa39b42e8f4d6923f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/a9862700a40f48dfa39b42e8f4d6923f 2024-11-15T22:16:23,753 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/a9862700a40f48dfa39b42e8f4d6923f, entries=150, sequenceid=428, filesize=30.5 K 2024-11-15T22:16:23,756 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/B/5c579e7d2e164c9e8b7b81ecf9f4803e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/5c579e7d2e164c9e8b7b81ecf9f4803e 2024-11-15T22:16:23,779 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/5c579e7d2e164c9e8b7b81ecf9f4803e, entries=150, sequenceid=428, filesize=12.0 K 2024-11-15T22:16:23,784 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/.tmp/C/f4f07fe3dde04f2fba16021cb97134ac as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/f4f07fe3dde04f2fba16021cb97134ac 2024-11-15T22:16:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T22:16:23,808 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/f4f07fe3dde04f2fba16021cb97134ac, entries=150, sequenceid=428, filesize=12.0 K 2024-11-15T22:16:23,813 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 7b87423906fae6354c3396098e18811c in 268ms, sequenceid=428, compaction requested=true 2024-11-15T22:16:23,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c3bf897b52cc4e4db55585a903806cf5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/fe583f7cfaab451f854d60b196277b1d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c41165631cdc46588fa02059fb8cfaae, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/9897286d002b402f990ab4d7f3bc63bd, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ee0907f4d7fb49ed9f9fb73235ed2c1f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ab8c57fb7db74d02b0d01a54a3ba1dc8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/06e31d4f67134ea5927ce2f41728b5ef, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ad186fbb88434bd39cb1009dd77c0601, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/8287d56eddcf46ee91b6d7263c171cca, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0077f5cf49cc47dfa9b9ac68c2b097f0, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/2bb476d9479d41c892d9528dbc2d8735, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/7543b6c0ce7e4122b203cbe77ee287e5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/36ff8c2bbcd94853aedcfd581feed2b2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/17382afc3bc742ba8e7d657216dffb40, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c080cfccdbee4a3b8cf5f7200fcd5d89, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/b082f7270b494d808572c0f67741db67, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/9df502db58cb4fd0be753876c974c924, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/77aef2e4de32452abb273062d083269f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/46ff3571a6e34f50ba1bea27ca25716d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/91b19f878b5641b4a8abd615a6a34e23, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/f855207503cd4732a7806eb3bfc1a809, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/8f0740ece25e4b46b9aef2035656e419, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0df882ccf89c47baa9192b9334f36924] to archive 2024-11-15T22:16:23,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:16:23,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c3bf897b52cc4e4db55585a903806cf5 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c3bf897b52cc4e4db55585a903806cf5 2024-11-15T22:16:23,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/fe583f7cfaab451f854d60b196277b1d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/fe583f7cfaab451f854d60b196277b1d 2024-11-15T22:16:23,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c41165631cdc46588fa02059fb8cfaae to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c41165631cdc46588fa02059fb8cfaae 2024-11-15T22:16:23,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/9897286d002b402f990ab4d7f3bc63bd to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/9897286d002b402f990ab4d7f3bc63bd 2024-11-15T22:16:23,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ee0907f4d7fb49ed9f9fb73235ed2c1f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ee0907f4d7fb49ed9f9fb73235ed2c1f 2024-11-15T22:16:23,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ab8c57fb7db74d02b0d01a54a3ba1dc8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ab8c57fb7db74d02b0d01a54a3ba1dc8 2024-11-15T22:16:23,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/06e31d4f67134ea5927ce2f41728b5ef to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/06e31d4f67134ea5927ce2f41728b5ef 2024-11-15T22:16:23,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ad186fbb88434bd39cb1009dd77c0601 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/ad186fbb88434bd39cb1009dd77c0601 2024-11-15T22:16:23,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/8287d56eddcf46ee91b6d7263c171cca to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/8287d56eddcf46ee91b6d7263c171cca 2024-11-15T22:16:23,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0077f5cf49cc47dfa9b9ac68c2b097f0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0077f5cf49cc47dfa9b9ac68c2b097f0 2024-11-15T22:16:23,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/2bb476d9479d41c892d9528dbc2d8735 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/2bb476d9479d41c892d9528dbc2d8735 2024-11-15T22:16:23,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/7543b6c0ce7e4122b203cbe77ee287e5 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/7543b6c0ce7e4122b203cbe77ee287e5 2024-11-15T22:16:23,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/36ff8c2bbcd94853aedcfd581feed2b2 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/36ff8c2bbcd94853aedcfd581feed2b2 2024-11-15T22:16:23,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/17382afc3bc742ba8e7d657216dffb40 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/17382afc3bc742ba8e7d657216dffb40 2024-11-15T22:16:23,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c080cfccdbee4a3b8cf5f7200fcd5d89 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/c080cfccdbee4a3b8cf5f7200fcd5d89 2024-11-15T22:16:23,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/b082f7270b494d808572c0f67741db67 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/b082f7270b494d808572c0f67741db67 2024-11-15T22:16:23,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/9df502db58cb4fd0be753876c974c924 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/9df502db58cb4fd0be753876c974c924 2024-11-15T22:16:23,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/77aef2e4de32452abb273062d083269f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/77aef2e4de32452abb273062d083269f 2024-11-15T22:16:23,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/46ff3571a6e34f50ba1bea27ca25716d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/46ff3571a6e34f50ba1bea27ca25716d 2024-11-15T22:16:23,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/91b19f878b5641b4a8abd615a6a34e23 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/91b19f878b5641b4a8abd615a6a34e23 2024-11-15T22:16:23,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/f855207503cd4732a7806eb3bfc1a809 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/f855207503cd4732a7806eb3bfc1a809 2024-11-15T22:16:23,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/8f0740ece25e4b46b9aef2035656e419 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/8f0740ece25e4b46b9aef2035656e419 2024-11-15T22:16:23,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0df882ccf89c47baa9192b9334f36924 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0df882ccf89c47baa9192b9334f36924 2024-11-15T22:16:23,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/67975ef7f21e4f9e81ae4685353ab9dc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/73dfb23a3d6f4bdfb9831e9c82d3e381, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/246cb621969444888ecfba228e7d9c87, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/0a7f64045def449aa44a4085c15638bd, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/d0a8df407ba445cc9e34f3e27a0f15e8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b531dd4b8679422aa907a2a908869324, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/8bf0205f09d34ae59850d0a1050e9ce7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/0b1bb52e8ba7468d84953ac090dd22ae, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/70eb828b120347da99bb15f39d9b3f28, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/4acdb7413d3642fe913a8c067303dfc1, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b07f1956ae1d49e3b3f66c3fc9732bfb, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/284d357a589747079d41a234b4ee31a2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/73ececb1305c4aa3b3939f93b3386561, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/ad469ce54bbc4eaa803c8236580bc101, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/9a5f91106654441a91c94914ca2d1448, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/31811d9fc0c44f2e89430cbf2492d39e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/7bcd65f0ca8c4d67b167579df412e81b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/343896cde95e4d69815c73df76e6ceae, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b0130503e70e46a99a29d04a4fe9b2f7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/c89aa883e5f0487b9a366931b0d5a92f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b767923cde46480c8d439409d13a2027, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/4c130364bd934cfca19fae417e19c979, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/db51e5575c4245b399833e92d465ac07] to archive 2024-11-15T22:16:23,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:16:23,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/67975ef7f21e4f9e81ae4685353ab9dc to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/67975ef7f21e4f9e81ae4685353ab9dc 2024-11-15T22:16:24,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/73dfb23a3d6f4bdfb9831e9c82d3e381 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/73dfb23a3d6f4bdfb9831e9c82d3e381 2024-11-15T22:16:24,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/246cb621969444888ecfba228e7d9c87 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/246cb621969444888ecfba228e7d9c87 2024-11-15T22:16:24,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/0a7f64045def449aa44a4085c15638bd to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/0a7f64045def449aa44a4085c15638bd 2024-11-15T22:16:24,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/d0a8df407ba445cc9e34f3e27a0f15e8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/d0a8df407ba445cc9e34f3e27a0f15e8 2024-11-15T22:16:24,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b531dd4b8679422aa907a2a908869324 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b531dd4b8679422aa907a2a908869324 2024-11-15T22:16:24,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/8bf0205f09d34ae59850d0a1050e9ce7 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/8bf0205f09d34ae59850d0a1050e9ce7 2024-11-15T22:16:24,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/0b1bb52e8ba7468d84953ac090dd22ae to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/0b1bb52e8ba7468d84953ac090dd22ae 2024-11-15T22:16:24,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/70eb828b120347da99bb15f39d9b3f28 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/70eb828b120347da99bb15f39d9b3f28 2024-11-15T22:16:24,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/4acdb7413d3642fe913a8c067303dfc1 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/4acdb7413d3642fe913a8c067303dfc1 2024-11-15T22:16:24,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b07f1956ae1d49e3b3f66c3fc9732bfb to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b07f1956ae1d49e3b3f66c3fc9732bfb 2024-11-15T22:16:24,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/284d357a589747079d41a234b4ee31a2 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/284d357a589747079d41a234b4ee31a2 2024-11-15T22:16:24,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/73ececb1305c4aa3b3939f93b3386561 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/73ececb1305c4aa3b3939f93b3386561 2024-11-15T22:16:24,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/ad469ce54bbc4eaa803c8236580bc101 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/ad469ce54bbc4eaa803c8236580bc101 2024-11-15T22:16:24,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/9a5f91106654441a91c94914ca2d1448 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/9a5f91106654441a91c94914ca2d1448 2024-11-15T22:16:24,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/31811d9fc0c44f2e89430cbf2492d39e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/31811d9fc0c44f2e89430cbf2492d39e 2024-11-15T22:16:24,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/7bcd65f0ca8c4d67b167579df412e81b to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/7bcd65f0ca8c4d67b167579df412e81b 2024-11-15T22:16:24,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/343896cde95e4d69815c73df76e6ceae to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/343896cde95e4d69815c73df76e6ceae 2024-11-15T22:16:24,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b0130503e70e46a99a29d04a4fe9b2f7 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b0130503e70e46a99a29d04a4fe9b2f7 2024-11-15T22:16:24,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/c89aa883e5f0487b9a366931b0d5a92f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/c89aa883e5f0487b9a366931b0d5a92f 2024-11-15T22:16:24,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b767923cde46480c8d439409d13a2027 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/b767923cde46480c8d439409d13a2027 2024-11-15T22:16:24,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/4c130364bd934cfca19fae417e19c979 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/4c130364bd934cfca19fae417e19c979 2024-11-15T22:16:24,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/db51e5575c4245b399833e92d465ac07 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/db51e5575c4245b399833e92d465ac07 2024-11-15T22:16:24,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/566d17d1c09e4a99911cbfeee3d8c3f4, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/4ccb2dbaf5a4474ea66970dc27f94632, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/7bf853aca4ca428493e6feff034e06fa, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/1e6a5aea32e2442c94c51f136d2bc726, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/9803126df3304ef3949aaae915b2334b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/9908bf093f734cd0a58666aecdeedb00, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/4297f1cb14ba47f294c15facf141e7ac, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/8aff3b1984c64dfaa433e778c8f0fcd4, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/3cd033baf1e147c79fbd5086ff04cff9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/940227fc62da4c74a1fd85ad27135cff, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/e4dd1eb7c67640dfb447b42593de12db, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/0e5b273218c94ffe95f9b3159fe7953f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/ddf4058cc9cc47deb62b49e73861e1bd, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/b40d94ca94ff47d080864dd15359b172, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/c02b7b4e94794fe5bcd400a7e86ee879, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d9402eefa4874f6e880738def8bbecbf, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d6a6542c8b57485d937a222aa787f0f9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/b908d3bf4d87486db3f322f423a166a7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/e548bcef7d524cb998b0aa850fbf35c2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d2ba7e814e474f228454efa1ee73e2ec, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/6a2d05f3c8024a52bf47aecf4b5fe4a3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/2e5284d314fb4618a2f18ad146a473cb, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/c2390ce3786c4e1a8fe6dbcbf73e9828] to archive 2024-11-15T22:16:24,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:16:24,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/566d17d1c09e4a99911cbfeee3d8c3f4 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/566d17d1c09e4a99911cbfeee3d8c3f4 2024-11-15T22:16:24,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/4ccb2dbaf5a4474ea66970dc27f94632 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/4ccb2dbaf5a4474ea66970dc27f94632 2024-11-15T22:16:24,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/7bf853aca4ca428493e6feff034e06fa to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/7bf853aca4ca428493e6feff034e06fa 2024-11-15T22:16:24,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/1e6a5aea32e2442c94c51f136d2bc726 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/1e6a5aea32e2442c94c51f136d2bc726 2024-11-15T22:16:24,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/9803126df3304ef3949aaae915b2334b to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/9803126df3304ef3949aaae915b2334b 2024-11-15T22:16:24,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/9908bf093f734cd0a58666aecdeedb00 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/9908bf093f734cd0a58666aecdeedb00 2024-11-15T22:16:24,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/4297f1cb14ba47f294c15facf141e7ac to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/4297f1cb14ba47f294c15facf141e7ac 2024-11-15T22:16:24,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/8aff3b1984c64dfaa433e778c8f0fcd4 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/8aff3b1984c64dfaa433e778c8f0fcd4 2024-11-15T22:16:24,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/3cd033baf1e147c79fbd5086ff04cff9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/3cd033baf1e147c79fbd5086ff04cff9 2024-11-15T22:16:24,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/940227fc62da4c74a1fd85ad27135cff to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/940227fc62da4c74a1fd85ad27135cff 2024-11-15T22:16:24,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/e4dd1eb7c67640dfb447b42593de12db to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/e4dd1eb7c67640dfb447b42593de12db 2024-11-15T22:16:24,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/0e5b273218c94ffe95f9b3159fe7953f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/0e5b273218c94ffe95f9b3159fe7953f 2024-11-15T22:16:24,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/ddf4058cc9cc47deb62b49e73861e1bd to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/ddf4058cc9cc47deb62b49e73861e1bd 2024-11-15T22:16:24,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/b40d94ca94ff47d080864dd15359b172 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/b40d94ca94ff47d080864dd15359b172 2024-11-15T22:16:24,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/c02b7b4e94794fe5bcd400a7e86ee879 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/c02b7b4e94794fe5bcd400a7e86ee879 2024-11-15T22:16:24,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d9402eefa4874f6e880738def8bbecbf to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d9402eefa4874f6e880738def8bbecbf 2024-11-15T22:16:24,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d6a6542c8b57485d937a222aa787f0f9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d6a6542c8b57485d937a222aa787f0f9 2024-11-15T22:16:24,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/b908d3bf4d87486db3f322f423a166a7 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/b908d3bf4d87486db3f322f423a166a7 2024-11-15T22:16:24,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/e548bcef7d524cb998b0aa850fbf35c2 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/e548bcef7d524cb998b0aa850fbf35c2 2024-11-15T22:16:24,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d2ba7e814e474f228454efa1ee73e2ec to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/d2ba7e814e474f228454efa1ee73e2ec 2024-11-15T22:16:24,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/6a2d05f3c8024a52bf47aecf4b5fe4a3 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/6a2d05f3c8024a52bf47aecf4b5fe4a3 2024-11-15T22:16:24,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/2e5284d314fb4618a2f18ad146a473cb to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/2e5284d314fb4618a2f18ad146a473cb 2024-11-15T22:16:24,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/c2390ce3786c4e1a8fe6dbcbf73e9828 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/c2390ce3786c4e1a8fe6dbcbf73e9828 2024-11-15T22:16:24,391 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/recovered.edits/431.seqid, newMaxSeqId=431, maxSeqId=4 2024-11-15T22:16:24,392 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. 2024-11-15T22:16:24,392 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for 7b87423906fae6354c3396098e18811c: Waiting for close lock at 1731708982840Running coprocessor pre-close hooks at 1731708982840Disabling compacts and flushes for region at 1731708982840Disabling writes for close at 1731708983544 (+704 ms)Obtaining lock to block concurrent updates at 1731708983545 (+1 ms)Preparing flush snapshotting stores in 7b87423906fae6354c3396098e18811c at 1731708983545Finished memstore snapshotting TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c., syncing WAL and waiting on mvcc, flushsize=dataSize=96180, getHeapSize=286320, getOffHeapSize=0, getCellsCount=2100 at 1731708983545Flushing stores of TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c. at 1731708983546 (+1 ms)Flushing 7b87423906fae6354c3396098e18811c/A: creating writer at 1731708983546Flushing mob file 7b87423906fae6354c3396098e18811c/A: appending metadata at 1731708983557 (+11 ms)Flushing mob file 7b87423906fae6354c3396098e18811c/A: closing flushed file at 1731708983557Flushing 7b87423906fae6354c3396098e18811c/A: appending metadata at 1731708983622 (+65 ms)Flushing 7b87423906fae6354c3396098e18811c/A: closing flushed file at 1731708983622Flushing 7b87423906fae6354c3396098e18811c/B: creating writer at 1731708983663 (+41 ms)Flushing 7b87423906fae6354c3396098e18811c/B: appending metadata at 1731708983690 (+27 ms)Flushing 7b87423906fae6354c3396098e18811c/B: closing flushed file at 1731708983690Flushing 7b87423906fae6354c3396098e18811c/C: creating writer at 1731708983709 (+19 ms)Flushing 7b87423906fae6354c3396098e18811c/C: appending metadata at 1731708983712 (+3 ms)Flushing 7b87423906fae6354c3396098e18811c/C: closing flushed file at 1731708983712Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46e54e84: reopening flushed file at 1731708983731 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71c72592: reopening flushed file at 1731708983754 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b1a3793: reopening flushed file at 1731708983779 (+25 ms)Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 7b87423906fae6354c3396098e18811c in 268ms, sequenceid=428, compaction requested=true at 1731708983813 (+34 ms)Writing region close event to WAL at 1731708984376 (+563 ms)Running coprocessor post-close hooks at 1731708984392 (+16 ms)Closed at 1731708984392 2024-11-15T22:16:24,396 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=7b87423906fae6354c3396098e18811c, regionState=CLOSED 2024-11-15T22:16:24,397 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed 7b87423906fae6354c3396098e18811c 2024-11-15T22:16:24,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:16:24,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-15T22:16:24,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure 7b87423906fae6354c3396098e18811c, server=e611192d6313,38779,1731708927034 in 1.7320 sec 2024-11-15T22:16:24,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-15T22:16:24,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7b87423906fae6354c3396098e18811c, UNASSIGN in 1.7470 sec 2024-11-15T22:16:24,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-15T22:16:24,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7540 sec 2024-11-15T22:16:24,439 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708984439"}]},"ts":"1731708984439"} 2024-11-15T22:16:24,442 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T22:16:24,442 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T22:16:24,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7890 sec 2024-11-15T22:16:24,650 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T22:16:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T22:16:24,812 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:24,812 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T22:16:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:24,814 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T22:16:24,816 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:24,824 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c 2024-11-15T22:16:24,845 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/recovered.edits] 2024-11-15T22:16:24,863 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0d13b808f29f4b56ac286e77cf26f11e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/0d13b808f29f4b56ac286e77cf26f11e 2024-11-15T22:16:24,865 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/7db1186f856341acaa462baafb1abea6 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/7db1186f856341acaa462baafb1abea6 2024-11-15T22:16:24,874 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/a9862700a40f48dfa39b42e8f4d6923f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/A/a9862700a40f48dfa39b42e8f4d6923f 2024-11-15T22:16:24,893 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/54ad03e8dd6d4e6e94ca60fd70bb2f5e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/54ad03e8dd6d4e6e94ca60fd70bb2f5e 2024-11-15T22:16:24,898 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/5c579e7d2e164c9e8b7b81ecf9f4803e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/5c579e7d2e164c9e8b7b81ecf9f4803e 2024-11-15T22:16:24,902 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/ba30b813023e46cb88dfa6d00434e8ba to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/B/ba30b813023e46cb88dfa6d00434e8ba 2024-11-15T22:16:24,905 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/40ab1e9026d74da8a509d33390003544 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/40ab1e9026d74da8a509d33390003544 2024-11-15T22:16:24,907 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/846ffb6e76c44545a9c1a10aeb4afcdd to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/846ffb6e76c44545a9c1a10aeb4afcdd 2024-11-15T22:16:24,910 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/f4f07fe3dde04f2fba16021cb97134ac to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/C/f4f07fe3dde04f2fba16021cb97134ac 2024-11-15T22:16:24,917 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/recovered.edits/431.seqid to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c/recovered.edits/431.seqid 2024-11-15T22:16:24,919 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/7b87423906fae6354c3396098e18811c 2024-11-15T22:16:24,919 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T22:16:24,919 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T22:16:24,921 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T22:16:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T22:16:24,929 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150a2cee14349d44109cf1c29c3a60ee94_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150a2cee14349d44109cf1c29c3a60ee94_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:24,931 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150cc4f066fe794575a75aba44a04e6ad0_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150cc4f066fe794575a75aba44a04e6ad0_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:24,934 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115352d42dfb2b34fac967f1ba15c0cd473_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115352d42dfb2b34fac967f1ba15c0cd473_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:24,936 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153533307c65674150b59d6a60cb95c89c_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153533307c65674150b59d6a60cb95c89c_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:24,939 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115353f31f43e244a7da986ef0d11f6d064_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115353f31f43e244a7da986ef0d11f6d064_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:24,967 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115498f2511303945128287e36de0e0d99b_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115498f2511303945128287e36de0e0d99b_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:24,989 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111556d3f3b4ff844233b9242a6334c61e9b_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111556d3f3b4ff844233b9242a6334c61e9b_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:25,013 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156d96790105fa4751a0c81efc0092b6ef_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156d96790105fa4751a0c81efc0092b6ef_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:25,014 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115856a3032bd0e4834b3a2edd9a7eb312f_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115856a3032bd0e4834b3a2edd9a7eb312f_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:25,021 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111588e9b020bc374a6e8ec045ac0df6a488_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111588e9b020bc374a6e8ec045ac0df6a488_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:25,025 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111595f9c67bdd5c426fb4f77438958d7fc3_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111595f9c67bdd5c426fb4f77438958d7fc3_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:25,027 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159bbc280c719149f2ac062ab838103230_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159bbc280c719149f2ac062ab838103230_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:25,028 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ac049f44ae77418c8119be151ec3f590_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ac049f44ae77418c8119be151ec3f590_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:25,029 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c26a2626f37d445bb01af9bede7473b6_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c26a2626f37d445bb01af9bede7473b6_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:25,041 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cbf7f60152d44f03962f718feb906679_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cbf7f60152d44f03962f718feb906679_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:25,061 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d8a6f1dc5e4147758174560fc7bf6151_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d8a6f1dc5e4147758174560fc7bf6151_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:25,064 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f5c239b2ad3840ec91bb6e89a51b180e_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f5c239b2ad3840ec91bb6e89a51b180e_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:25,077 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fdafec0c83054b588a3862fc1bb69d42_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fdafec0c83054b588a3862fc1bb69d42_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:25,085 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115feea48df6fac49d8a50972537ef8ed4b_7b87423906fae6354c3396098e18811c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115feea48df6fac49d8a50972537ef8ed4b_7b87423906fae6354c3396098e18811c 2024-11-15T22:16:25,086 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T22:16:25,111 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:25,114 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T22:16:25,123 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T22:16:25,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T22:16:25,133 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:25,133 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T22:16:25,133 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731708985133"}]},"ts":"9223372036854775807"} 2024-11-15T22:16:25,143 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T22:16:25,143 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7b87423906fae6354c3396098e18811c, NAME => 'TestAcidGuarantees,,1731708959263.7b87423906fae6354c3396098e18811c.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T22:16:25,144 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T22:16:25,144 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731708985144"}]},"ts":"9223372036854775807"} 2024-11-15T22:16:25,157 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T22:16:25,161 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:25,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 349 msec 2024-11-15T22:16:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T22:16:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T22:16:25,445 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:25,466 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 250) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1052444504_22 at /127.0.0.1:58448 [Waiting for operation #278] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/cluster_a6235007-a423-ea83-cf16-0fd92ac68712/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/cluster_a6235007-a423-ea83-cf16-0fd92ac68712/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-556696224_22 at /127.0.0.1:58456 [Waiting for operation #269] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1052444504_22 at /127.0.0.1:58398 [Waiting for operation #284] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-556696224_22 at /127.0.0.1:56020 [Waiting for operation #576] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1053 (was 859) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=246 (was 1411) 2024-11-15T22:16:25,480 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=1053, ProcessCount=11, AvailableMemoryMB=246 2024-11-15T22:16:25,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:16:25,482 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:16:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:25,489 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T22:16:25,489 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:25,490 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-11-15T22:16:25,491 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T22:16:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T22:16:25,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742061_1237 (size=1061) 2024-11-15T22:16:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T22:16:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T22:16:25,911 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e 2024-11-15T22:16:25,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742062_1238 (size=53) 2024-11-15T22:16:25,917 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:16:25,917 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c203d77ebc02584fa5952f844fc15b8c, disabling compactions & flushes 2024-11-15T22:16:25,917 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:25,917 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:25,917 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. after waiting 0 ms 2024-11-15T22:16:25,917 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:25,917 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:25,917 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c203d77ebc02584fa5952f844fc15b8c: Waiting for close lock at 1731708985917Disabling compacts and flushes for region at 1731708985917Disabling writes for close at 1731708985917Writing region close event to WAL at 1731708985917Closed at 1731708985917 2024-11-15T22:16:25,918 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T22:16:25,918 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731708985918"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708985918"}]},"ts":"1731708985918"} 2024-11-15T22:16:25,920 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T22:16:25,921 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T22:16:25,921 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708985921"}]},"ts":"1731708985921"} 2024-11-15T22:16:25,923 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T22:16:25,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c203d77ebc02584fa5952f844fc15b8c, ASSIGN}] 2024-11-15T22:16:25,924 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c203d77ebc02584fa5952f844fc15b8c, ASSIGN 2024-11-15T22:16:25,925 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c203d77ebc02584fa5952f844fc15b8c, ASSIGN; state=OFFLINE, location=e611192d6313,38779,1731708927034; forceNewPlan=false, retain=false 2024-11-15T22:16:26,075 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=c203d77ebc02584fa5952f844fc15b8c, regionState=OPENING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:16:26,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c203d77ebc02584fa5952f844fc15b8c, ASSIGN because future has completed 2024-11-15T22:16:26,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:16:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T22:16:26,233 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:26,233 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:16:26,233 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:26,233 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:16:26,233 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:26,233 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:26,235 INFO [StoreOpener-c203d77ebc02584fa5952f844fc15b8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:26,236 INFO [StoreOpener-c203d77ebc02584fa5952f844fc15b8c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:16:26,236 INFO [StoreOpener-c203d77ebc02584fa5952f844fc15b8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c203d77ebc02584fa5952f844fc15b8c columnFamilyName A 2024-11-15T22:16:26,236 DEBUG [StoreOpener-c203d77ebc02584fa5952f844fc15b8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:26,236 INFO [StoreOpener-c203d77ebc02584fa5952f844fc15b8c-1 {}] regionserver.HStore(327): Store=c203d77ebc02584fa5952f844fc15b8c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:16:26,236 INFO [StoreOpener-c203d77ebc02584fa5952f844fc15b8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:26,249 INFO [StoreOpener-c203d77ebc02584fa5952f844fc15b8c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:16:26,249 INFO [StoreOpener-c203d77ebc02584fa5952f844fc15b8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c203d77ebc02584fa5952f844fc15b8c columnFamilyName B 2024-11-15T22:16:26,249 DEBUG [StoreOpener-c203d77ebc02584fa5952f844fc15b8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:26,250 INFO [StoreOpener-c203d77ebc02584fa5952f844fc15b8c-1 {}] regionserver.HStore(327): Store=c203d77ebc02584fa5952f844fc15b8c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:16:26,250 INFO [StoreOpener-c203d77ebc02584fa5952f844fc15b8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:26,251 INFO [StoreOpener-c203d77ebc02584fa5952f844fc15b8c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:16:26,251 INFO [StoreOpener-c203d77ebc02584fa5952f844fc15b8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c203d77ebc02584fa5952f844fc15b8c columnFamilyName C 2024-11-15T22:16:26,252 DEBUG [StoreOpener-c203d77ebc02584fa5952f844fc15b8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:26,252 INFO [StoreOpener-c203d77ebc02584fa5952f844fc15b8c-1 {}] regionserver.HStore(327): Store=c203d77ebc02584fa5952f844fc15b8c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:16:26,252 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:26,252 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:26,253 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:26,253 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:26,254 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:26,254 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:26,254 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:16:26,256 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:26,257 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:16:26,258 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened c203d77ebc02584fa5952f844fc15b8c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69925785, jitterRate=0.041975393891334534}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:16:26,258 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:26,258 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for c203d77ebc02584fa5952f844fc15b8c: Running coprocessor pre-open hook at 1731708986233Writing region info on filesystem at 1731708986233Initializing all the Stores at 1731708986234 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708986234Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708986234Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708986235 (+1 ms)Cleaning up temporary data from old regions at 1731708986254 (+19 ms)Running coprocessor post-open hooks at 1731708986258 (+4 ms)Region opened successfully at 1731708986258 2024-11-15T22:16:26,259 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., pid=57, masterSystemTime=1731708986230 2024-11-15T22:16:26,261 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:26,261 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:26,261 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=c203d77ebc02584fa5952f844fc15b8c, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:16:26,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:16:26,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-15T22:16:26,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 in 186 msec 2024-11-15T22:16:26,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-15T22:16:26,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c203d77ebc02584fa5952f844fc15b8c, ASSIGN in 342 msec 2024-11-15T22:16:26,268 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T22:16:26,268 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708986268"}]},"ts":"1731708986268"} 2024-11-15T22:16:26,269 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T22:16:26,270 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T22:16:26,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 788 msec 2024-11-15T22:16:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T22:16:26,632 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:26,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5832a527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:26,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:26,634 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24482055, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:26,636 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f94bc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,637 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:26,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:26,640 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T22:16:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is e611192d6313,34969,1731708925722 2024-11-15T22:16:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3e0f0acc 2024-11-15T22:16:26,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T22:16:26,642 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T22:16:26,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a08a8a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:26,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:26,644 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11d3e074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:26,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:26,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:26,645 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f38c060, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ddfbea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,647 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:26,647 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:26,648 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@81dacfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:26,650 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:26,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ddbb1d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fbbb442, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:26,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:26,653 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:26,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:26,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:26,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c4ab85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:26,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:26,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:26,655 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:26,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@324b6329, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341051a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:26,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:26,658 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@235dafc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:26,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:26,659 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:26,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56960a23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309aa0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:26,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:26,662 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e70a5af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:26,664 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72a84591, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b624df0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:26,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:26,669 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38c2e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:26,671 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aa1778c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@106f6d84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:26,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:26,682 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49bc6b12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:26,683 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:26,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e933378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a00a41f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:26,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:26,687 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e5c3c57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:26,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:26,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:26,689 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:26,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22501971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a680e8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:26,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:26,692 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c3e16fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:26,695 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:26,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77df1bcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292fd0b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,700 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:26,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:26,701 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3b8ba6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:26,703 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d4597d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:26,705 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:26,706 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:26,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:26,708 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:26,709 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:26,709 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-15T22:16:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:16:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T22:16:26,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:26,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:26,711 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:26,713 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:26,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:26,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:26,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:26,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:16:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:16:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:16:26,719 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:26,720 DEBUG [Thread-1124 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:26,720 DEBUG [Thread-1124 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:26,722 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:26,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:16:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:26,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T22:16:26,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:26,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:26,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:26,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:26,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:26,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:26,739 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:26,740 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:26,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:26,741 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:26,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:26,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:26,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:26,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:16:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:16:26,753 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:26,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:26,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:26,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:26,756 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:26,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:16:26,766 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:26,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:26,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:26,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:26,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:26,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:26,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:16:26,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:16:26,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/1497b0f6a81d4d9ba08bff7e0edf7228 is 50, key is test_row_0/A:col10/1731708986735/Put/seqid=0 2024-11-15T22:16:26,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742063_1239 (size=9657) 2024-11-15T22:16:26,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731708986793, max timestamp=1731708986809Num uniques 859; 2024-11-15T22:16:26,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731708986793, max timestamp=1731708986809Num uniques 859; 2024-11-15T22:16:26,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T22:16:26,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T22:16:26,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/1497b0f6a81d4d9ba08bff7e0edf7228 2024-11-15T22:16:26,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731708986793, max timestamp=1731708986809Num uniques 859; 2024-11-15T22:16:26,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T22:16:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T22:16:26,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:26,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:26,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:26,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:26,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:26,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:26,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/ea9cec487be24ac0b76ba9bc7056de6b is 50, key is test_row_0/B:col10/1731708986735/Put/seqid=0 2024-11-15T22:16:26,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709046888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:26,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709046896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:26,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709046903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:26,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709046904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:26,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709046913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:26,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742064_1240 (size=9657) 2024-11-15T22:16:26,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/ea9cec487be24ac0b76ba9bc7056de6b 2024-11-15T22:16:27,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/544b331b497b427ead789d7e36df814c is 50, key is test_row_0/C:col10/1731708986735/Put/seqid=0 2024-11-15T22:16:27,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709047012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709047013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709047022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709047022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T22:16:27,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709047033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:27,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:27,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:27,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:27,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742065_1241 (size=9657) 2024-11-15T22:16:27,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:27,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:27,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:27,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:27,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709047224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709047228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709047231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709047244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709047253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T22:16:27,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:27,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:27,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:27,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:27,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/544b331b497b427ead789d7e36df814c 2024-11-15T22:16:27,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/1497b0f6a81d4d9ba08bff7e0edf7228 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/1497b0f6a81d4d9ba08bff7e0edf7228 2024-11-15T22:16:27,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/1497b0f6a81d4d9ba08bff7e0edf7228, entries=100, sequenceid=12, filesize=9.4 K 2024-11-15T22:16:27,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/ea9cec487be24ac0b76ba9bc7056de6b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/ea9cec487be24ac0b76ba9bc7056de6b 2024-11-15T22:16:27,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/ea9cec487be24ac0b76ba9bc7056de6b, entries=100, sequenceid=12, filesize=9.4 K 2024-11-15T22:16:27,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:27,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/544b331b497b427ead789d7e36df814c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/544b331b497b427ead789d7e36df814c 2024-11-15T22:16:27,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/544b331b497b427ead789d7e36df814c, entries=100, sequenceid=12, filesize=9.4 K 2024-11-15T22:16:27,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for c203d77ebc02584fa5952f844fc15b8c in 799ms, sequenceid=12, compaction requested=false 2024-11-15T22:16:27,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:27,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-15T22:16:27,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:27,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:27,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:27,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:27,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:27,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:27,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/36ab7b2464de436e929c67606deb5606 is 50, key is test_row_0/A:col10/1731708987553/Put/seqid=0 2024-11-15T22:16:27,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709047572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709047573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709047577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709047578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709047581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742066_1242 (size=12001) 2024-11-15T22:16:27,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:27,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:27,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:27,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:27,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709047691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709047692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709047692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709047701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:27,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:27,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:27,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:27,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T22:16:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709047912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709047913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709047913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709047922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:27,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:27,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:27,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:27,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:27,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:27,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:27,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/36ab7b2464de436e929c67606deb5606 2024-11-15T22:16:28,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/bbe92d11a8094071b4b5298492630402 is 50, key is test_row_0/B:col10/1731708987553/Put/seqid=0 2024-11-15T22:16:28,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709048092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:28,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742067_1243 (size=12001) 2024-11-15T22:16:28,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:28,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:28,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:28,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:28,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709048222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:28,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709048225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:28,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:28,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:28,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709048231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:28,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709048231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:28,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:28,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:28,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:28,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:28,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:28,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:28,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:28,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:28,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/bbe92d11a8094071b4b5298492630402 2024-11-15T22:16:28,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/f783ba49100642bfb050d091f40e1a38 is 50, key is test_row_0/C:col10/1731708987553/Put/seqid=0 2024-11-15T22:16:28,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742068_1244 (size=12001) 2024-11-15T22:16:28,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/f783ba49100642bfb050d091f40e1a38 2024-11-15T22:16:28,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/36ab7b2464de436e929c67606deb5606 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/36ab7b2464de436e929c67606deb5606 2024-11-15T22:16:28,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/36ab7b2464de436e929c67606deb5606, entries=150, sequenceid=49, filesize=11.7 K 2024-11-15T22:16:28,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/bbe92d11a8094071b4b5298492630402 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/bbe92d11a8094071b4b5298492630402 2024-11-15T22:16:28,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/bbe92d11a8094071b4b5298492630402, entries=150, sequenceid=49, filesize=11.7 K 2024-11-15T22:16:28,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:28,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:28,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:28,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:28,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/f783ba49100642bfb050d091f40e1a38 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f783ba49100642bfb050d091f40e1a38 2024-11-15T22:16:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f783ba49100642bfb050d091f40e1a38, entries=150, sequenceid=49, filesize=11.7 K 2024-11-15T22:16:28,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for c203d77ebc02584fa5952f844fc15b8c in 1075ms, sequenceid=49, compaction requested=false 2024-11-15T22:16:28,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:28,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T22:16:28,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:28,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:28,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:28,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:28,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:28,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:28,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/21b44306ab5a409d99ae8b86dfa81249 is 50, key is test_row_0/A:col10/1731708988737/Put/seqid=0 2024-11-15T22:16:28,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742069_1245 (size=14341) 2024-11-15T22:16:28,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:28,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:28,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:28,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:28,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731708988787, max timestamp=1731708988806Num uniques 859; 2024-11-15T22:16:28,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:16:28,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731708988787, max timestamp=1731708988806Num uniques 859; 2024-11-15T22:16:28,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:16:28,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731708988787, max timestamp=1731708988806Num uniques 859; 2024-11-15T22:16:28,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:16:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T22:16:28,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709048849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:28,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:28,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709048866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:28,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709048866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:28,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709048866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:28,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:28,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:28,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:28,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:28,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:28,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709048971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:28,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709048982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:28,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709048982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:29,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709048995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:29,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:29,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:29,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:29,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:29,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709049111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:29,119 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in c203d77ebc02584fa5952f844fc15b8c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:16:29,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:29,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/21b44306ab5a409d99ae8b86dfa81249 2024-11-15T22:16:29,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709049181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:29,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709049192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:29,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709049193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:29,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/16d3db7fb8814ccd9e95b2fe5106c67a is 50, key is test_row_0/B:col10/1731708988737/Put/seqid=0 2024-11-15T22:16:29,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709049211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:29,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742070_1246 (size=12001) 2024-11-15T22:16:29,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:29,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:29,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:29,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:29,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:29,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709049501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:29,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709049511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:29,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:29,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709049511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:29,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:29,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:29,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709049522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:29,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:29,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:29,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:29,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:29,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/16d3db7fb8814ccd9e95b2fe5106c67a 2024-11-15T22:16:29,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/e7e05e1fd0934713b7d5353bf6db7414 is 50, key is test_row_0/C:col10/1731708988737/Put/seqid=0 2024-11-15T22:16:29,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742071_1247 (size=12001) 2024-11-15T22:16:29,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:29,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:29,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:29,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:29,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:29,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:29,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:29,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:29,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:29,922 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T22:16:30,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709050011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:30,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709050021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:30,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709050022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:30,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709050031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:30,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:30,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:30,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:30,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:30,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:30,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:30,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/e7e05e1fd0934713b7d5353bf6db7414 2024-11-15T22:16:30,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/21b44306ab5a409d99ae8b86dfa81249 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/21b44306ab5a409d99ae8b86dfa81249 2024-11-15T22:16:30,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/21b44306ab5a409d99ae8b86dfa81249, entries=200, sequenceid=59, filesize=14.0 K 2024-11-15T22:16:30,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/16d3db7fb8814ccd9e95b2fe5106c67a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/16d3db7fb8814ccd9e95b2fe5106c67a 2024-11-15T22:16:30,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/16d3db7fb8814ccd9e95b2fe5106c67a, entries=150, sequenceid=59, filesize=11.7 K 2024-11-15T22:16:30,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/e7e05e1fd0934713b7d5353bf6db7414 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e7e05e1fd0934713b7d5353bf6db7414 2024-11-15T22:16:30,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e7e05e1fd0934713b7d5353bf6db7414, entries=150, sequenceid=59, filesize=11.7 K 2024-11-15T22:16:30,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c203d77ebc02584fa5952f844fc15b8c in 1392ms, sequenceid=59, compaction requested=true 2024-11-15T22:16:30,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:30,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:30,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:30,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:30,130 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:30,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:30,130 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:30,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:30,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:30,132 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:30,132 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/B is initiating minor compaction (all files) 2024-11-15T22:16:30,132 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/B in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:30,132 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/ea9cec487be24ac0b76ba9bc7056de6b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/bbe92d11a8094071b4b5298492630402, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/16d3db7fb8814ccd9e95b2fe5106c67a] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=32.9 K 2024-11-15T22:16:30,133 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:30,133 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/A is initiating minor compaction (all files) 2024-11-15T22:16:30,133 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/A in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:30,133 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/1497b0f6a81d4d9ba08bff7e0edf7228, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/36ab7b2464de436e929c67606deb5606, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/21b44306ab5a409d99ae8b86dfa81249] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=35.2 K 2024-11-15T22:16:30,133 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1497b0f6a81d4d9ba08bff7e0edf7228, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731708986733 2024-11-15T22:16:30,133 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting ea9cec487be24ac0b76ba9bc7056de6b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731708986733 2024-11-15T22:16:30,133 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36ab7b2464de436e929c67606deb5606, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731708986898 2024-11-15T22:16:30,133 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting bbe92d11a8094071b4b5298492630402, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731708986898 2024-11-15T22:16:30,134 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21b44306ab5a409d99ae8b86dfa81249, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731708987566 2024-11-15T22:16:30,134 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 16d3db7fb8814ccd9e95b2fe5106c67a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731708987566 2024-11-15T22:16:30,147 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#B#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:30,147 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/dda22b8703e746f7bb252e23ae6a232e is 50, key is test_row_0/B:col10/1731708988737/Put/seqid=0 2024-11-15T22:16:30,158 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#A#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:30,159 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/0e7f2a42e42f41448a00295e9e8be52c is 50, key is test_row_0/A:col10/1731708988737/Put/seqid=0 2024-11-15T22:16:30,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742072_1248 (size=12104) 2024-11-15T22:16:30,176 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/dda22b8703e746f7bb252e23ae6a232e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/dda22b8703e746f7bb252e23ae6a232e 2024-11-15T22:16:30,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742073_1249 (size=12104) 2024-11-15T22:16:30,189 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/B of c203d77ebc02584fa5952f844fc15b8c into dda22b8703e746f7bb252e23ae6a232e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:30,189 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:30,189 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/B, priority=13, startTime=1731708990130; duration=0sec 2024-11-15T22:16:30,189 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:30,189 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:B 2024-11-15T22:16:30,189 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:30,192 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:30,192 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/C is initiating minor compaction (all files) 2024-11-15T22:16:30,192 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/C in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:30,192 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/544b331b497b427ead789d7e36df814c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f783ba49100642bfb050d091f40e1a38, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e7e05e1fd0934713b7d5353bf6db7414] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=32.9 K 2024-11-15T22:16:30,193 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 544b331b497b427ead789d7e36df814c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731708986733 2024-11-15T22:16:30,194 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting f783ba49100642bfb050d091f40e1a38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731708986898 2024-11-15T22:16:30,194 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting e7e05e1fd0934713b7d5353bf6db7414, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731708987566 2024-11-15T22:16:30,195 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/0e7f2a42e42f41448a00295e9e8be52c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0e7f2a42e42f41448a00295e9e8be52c 2024-11-15T22:16:30,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T22:16:30,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:30,197 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-15T22:16:30,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:30,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:30,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:30,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:30,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:30,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:30,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/0b9e0cddf8b24f4d847d9bf74589df3f is 50, key is test_row_0/A:col10/1731708988857/Put/seqid=0 2024-11-15T22:16:30,210 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#C#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:30,211 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/5898c078c91345e89bf1055ac3b1c3ae is 50, key is test_row_0/C:col10/1731708988737/Put/seqid=0 2024-11-15T22:16:30,216 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/A of c203d77ebc02584fa5952f844fc15b8c into 0e7f2a42e42f41448a00295e9e8be52c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:30,216 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:30,216 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/A, priority=13, startTime=1731708990130; duration=0sec 2024-11-15T22:16:30,216 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:30,216 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:A 2024-11-15T22:16:30,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742074_1250 (size=12104) 2024-11-15T22:16:30,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742075_1251 (size=12001) 2024-11-15T22:16:30,229 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/0b9e0cddf8b24f4d847d9bf74589df3f 2024-11-15T22:16:30,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/1ec6987f32fc4f65b9017c120538a8bc is 50, key is test_row_0/B:col10/1731708988857/Put/seqid=0 2024-11-15T22:16:30,246 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/5898c078c91345e89bf1055ac3b1c3ae as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/5898c078c91345e89bf1055ac3b1c3ae 2024-11-15T22:16:30,255 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/C of c203d77ebc02584fa5952f844fc15b8c into 5898c078c91345e89bf1055ac3b1c3ae(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:30,255 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:30,255 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/C, priority=13, startTime=1731708990130; duration=0sec 2024-11-15T22:16:30,255 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:30,255 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:C 2024-11-15T22:16:30,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742076_1252 (size=12001) 2024-11-15T22:16:30,277 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/1ec6987f32fc4f65b9017c120538a8bc 2024-11-15T22:16:30,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/1191eba62a1f4d01968db8c913187470 is 50, key is test_row_0/C:col10/1731708988857/Put/seqid=0 2024-11-15T22:16:30,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742077_1253 (size=12001) 2024-11-15T22:16:30,313 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/1191eba62a1f4d01968db8c913187470 2024-11-15T22:16:30,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/0b9e0cddf8b24f4d847d9bf74589df3f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0b9e0cddf8b24f4d847d9bf74589df3f 2024-11-15T22:16:30,326 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0b9e0cddf8b24f4d847d9bf74589df3f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-15T22:16:30,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/1ec6987f32fc4f65b9017c120538a8bc as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/1ec6987f32fc4f65b9017c120538a8bc 2024-11-15T22:16:30,341 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/1ec6987f32fc4f65b9017c120538a8bc, entries=150, sequenceid=96, filesize=11.7 K 2024-11-15T22:16:30,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/1191eba62a1f4d01968db8c913187470 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/1191eba62a1f4d01968db8c913187470 2024-11-15T22:16:30,354 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/1191eba62a1f4d01968db8c913187470, entries=150, sequenceid=96, filesize=11.7 K 2024-11-15T22:16:30,355 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for c203d77ebc02584fa5952f844fc15b8c in 158ms, sequenceid=96, compaction requested=false 2024-11-15T22:16:30,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:30,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:30,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-15T22:16:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-15T22:16:30,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-15T22:16:30,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6430 sec 2024-11-15T22:16:30,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 3.6520 sec 2024-11-15T22:16:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T22:16:30,881 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:30,882 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-15T22:16:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T22:16:30,884 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:30,886 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:30,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T22:16:31,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-15T22:16:31,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:31,044 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-15T22:16:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:31,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:31,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/0f939796012b42709005fcb95cfa39ee is 50, key is test_row_0/A:col10/1731708991038/Put/seqid=0 2024-11-15T22:16:31,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742078_1254 (size=12001) 2024-11-15T22:16:31,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708991056, max timestamp=1731708991090Num uniques 859; 2024-11-15T22:16:31,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:16:31,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708991056, max timestamp=1731708991090Num uniques 859; 2024-11-15T22:16:31,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731708991056, max timestamp=1731708991090Num uniques 859; 2024-11-15T22:16:31,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:16:31,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:16:31,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731708991165, max timestamp=1731708991193Num uniques 1018; 2024-11-15T22:16:31,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731708991165, max timestamp=1731708991193Num uniques 1018; 2024-11-15T22:16:31,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-15T22:16:31,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731708991165, max timestamp=1731708991193Num uniques 1018; 2024-11-15T22:16:31,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-15T22:16:31,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-15T22:16:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T22:16:31,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709051216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731708991206, max timestamp=1731708991222Num uniques 1027; 2024-11-15T22:16:31,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-15T22:16:31,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731708991206, max timestamp=1731708991222Num uniques 1027; 2024-11-15T22:16:31,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731708991206, max timestamp=1731708991222Num uniques 1027; 2024-11-15T22:16:31,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-15T22:16:31,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-15T22:16:31,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709051253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709051254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709051258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709051260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709051331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709051372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709051372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709051372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709051381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,494 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/0f939796012b42709005fcb95cfa39ee 2024-11-15T22:16:31,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/b307ff3f90df425aaeb06fa3fa869ca8 is 50, key is test_row_0/B:col10/1731708991038/Put/seqid=0 2024-11-15T22:16:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T22:16:31,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742079_1255 (size=12001) 2024-11-15T22:16:31,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709051541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709051598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709051598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709051600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709051600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709051852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709051913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709051913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709051913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:31,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709051913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:31,925 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/b307ff3f90df425aaeb06fa3fa869ca8 2024-11-15T22:16:31,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/20c708165c0044cda75c25bdb0ebcd93 is 50, key is test_row_0/C:col10/1731708991038/Put/seqid=0 2024-11-15T22:16:31,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742080_1256 (size=12001) 2024-11-15T22:16:31,975 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/20c708165c0044cda75c25bdb0ebcd93 2024-11-15T22:16:31,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/0f939796012b42709005fcb95cfa39ee as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0f939796012b42709005fcb95cfa39ee 2024-11-15T22:16:31,988 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0f939796012b42709005fcb95cfa39ee, entries=150, sequenceid=106, filesize=11.7 K 2024-11-15T22:16:31,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/b307ff3f90df425aaeb06fa3fa869ca8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/b307ff3f90df425aaeb06fa3fa869ca8 2024-11-15T22:16:32,017 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/b307ff3f90df425aaeb06fa3fa869ca8, entries=150, sequenceid=106, filesize=11.7 K 2024-11-15T22:16:32,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/20c708165c0044cda75c25bdb0ebcd93 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/20c708165c0044cda75c25bdb0ebcd93 2024-11-15T22:16:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T22:16:32,022 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/20c708165c0044cda75c25bdb0ebcd93, entries=150, sequenceid=106, filesize=11.7 K 2024-11-15T22:16:32,024 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for c203d77ebc02584fa5952f844fc15b8c in 980ms, sequenceid=106, compaction requested=true 2024-11-15T22:16:32,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:32,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:32,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-15T22:16:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-15T22:16:32,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-15T22:16:32,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1450 sec 2024-11-15T22:16:32,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.1520 sec 2024-11-15T22:16:32,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=154.31 KB heapSize=434.39 KB 2024-11-15T22:16:32,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:32,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T22:16:32,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:32,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T22:16:32,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:32,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T22:16:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:32,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/f82dfd346d5849e48bfc3a58094cd31c is 50, key is test_row_0/A:col10/1731708992372/Put/seqid=0 2024-11-15T22:16:32,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742081_1257 (size=14541) 2024-11-15T22:16:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709052412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:32,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709052424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:32,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:32,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:32,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:32,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:32,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709052424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:32,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:32,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709052424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:32,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:32,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709052426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:32,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:32,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709052531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:32,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:32,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709052741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:32,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/f82dfd346d5849e48bfc3a58094cd31c 2024-11-15T22:16:32,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/8014abf52a98485db86b5e3af8935fc1 is 50, key is test_row_0/B:col10/1731708992372/Put/seqid=0 2024-11-15T22:16:32,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742082_1258 (size=12151) 2024-11-15T22:16:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T22:16:33,032 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:33,033 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-15T22:16:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T22:16:33,035 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:33,037 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:33,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:33,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709053051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:33,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T22:16:33,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T22:16:33,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:33,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:33,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:33,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:33,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:33,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:33,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/8014abf52a98485db86b5e3af8935fc1 2024-11-15T22:16:33,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/f5a986d2b0d54877881cc181eea1e2f6 is 50, key is test_row_0/C:col10/1731708992372/Put/seqid=0 2024-11-15T22:16:33,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742083_1259 (size=12151) 2024-11-15T22:16:33,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/f5a986d2b0d54877881cc181eea1e2f6 2024-11-15T22:16:33,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/f82dfd346d5849e48bfc3a58094cd31c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/f82dfd346d5849e48bfc3a58094cd31c 2024-11-15T22:16:33,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/f82dfd346d5849e48bfc3a58094cd31c, entries=200, sequenceid=178, filesize=14.2 K 2024-11-15T22:16:33,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/8014abf52a98485db86b5e3af8935fc1 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/8014abf52a98485db86b5e3af8935fc1 2024-11-15T22:16:33,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/8014abf52a98485db86b5e3af8935fc1, entries=150, sequenceid=178, filesize=11.9 K 2024-11-15T22:16:33,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/f5a986d2b0d54877881cc181eea1e2f6 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f5a986d2b0d54877881cc181eea1e2f6 2024-11-15T22:16:33,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f5a986d2b0d54877881cc181eea1e2f6, entries=150, sequenceid=178, filesize=11.9 K 2024-11-15T22:16:33,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~434.34 KB/444768, currentSize=26.84 KB/27480 for c203d77ebc02584fa5952f844fc15b8c in 922ms, sequenceid=178, compaction requested=true 2024-11-15T22:16:33,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:33,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:33,296 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:16:33,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:33,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:33,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:33,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:33,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:33,296 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:16:33,297 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50647 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:16:33,297 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:16:33,297 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/A is initiating minor compaction (all files) 2024-11-15T22:16:33,297 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/B is initiating minor compaction (all files) 2024-11-15T22:16:33,297 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/A in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:33,297 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/B in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:33,297 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/dda22b8703e746f7bb252e23ae6a232e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/1ec6987f32fc4f65b9017c120538a8bc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/b307ff3f90df425aaeb06fa3fa869ca8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/8014abf52a98485db86b5e3af8935fc1] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=47.1 K 2024-11-15T22:16:33,297 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0e7f2a42e42f41448a00295e9e8be52c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0b9e0cddf8b24f4d847d9bf74589df3f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0f939796012b42709005fcb95cfa39ee, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/f82dfd346d5849e48bfc3a58094cd31c] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=49.5 K 2024-11-15T22:16:33,297 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e7f2a42e42f41448a00295e9e8be52c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731708987566 2024-11-15T22:16:33,297 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting dda22b8703e746f7bb252e23ae6a232e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731708987566 2024-11-15T22:16:33,298 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ec6987f32fc4f65b9017c120538a8bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731708988842 2024-11-15T22:16:33,298 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b9e0cddf8b24f4d847d9bf74589df3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731708988842 2024-11-15T22:16:33,299 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting b307ff3f90df425aaeb06fa3fa869ca8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731708991038 2024-11-15T22:16:33,299 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f939796012b42709005fcb95cfa39ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731708991038 2024-11-15T22:16:33,299 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 8014abf52a98485db86b5e3af8935fc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731708991248 2024-11-15T22:16:33,299 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting f82dfd346d5849e48bfc3a58094cd31c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731708991248 2024-11-15T22:16:33,310 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#B#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:33,311 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/77a2a3bb47854884aece3492f31ca8bd is 50, key is test_row_0/B:col10/1731708992372/Put/seqid=0 2024-11-15T22:16:33,314 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#A#compaction#217 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:33,315 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/b0c6013924f24ea9bb06b9085a4ebf8b is 50, key is test_row_0/A:col10/1731708992372/Put/seqid=0 2024-11-15T22:16:33,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742084_1260 (size=12391) 2024-11-15T22:16:33,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742085_1261 (size=12391) 2024-11-15T22:16:33,337 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/b0c6013924f24ea9bb06b9085a4ebf8b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/b0c6013924f24ea9bb06b9085a4ebf8b 2024-11-15T22:16:33,342 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/A of c203d77ebc02584fa5952f844fc15b8c into b0c6013924f24ea9bb06b9085a4ebf8b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:33,343 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:33,343 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/A, priority=12, startTime=1731708993295; duration=0sec 2024-11-15T22:16:33,343 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:33,343 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:A 2024-11-15T22:16:33,343 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:16:33,344 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:16:33,344 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/C is initiating minor compaction (all files) 2024-11-15T22:16:33,344 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/C in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:33,344 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/5898c078c91345e89bf1055ac3b1c3ae, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/1191eba62a1f4d01968db8c913187470, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/20c708165c0044cda75c25bdb0ebcd93, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f5a986d2b0d54877881cc181eea1e2f6] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=47.1 K 2024-11-15T22:16:33,345 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5898c078c91345e89bf1055ac3b1c3ae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731708987566 2024-11-15T22:16:33,345 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1191eba62a1f4d01968db8c913187470, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731708988842 2024-11-15T22:16:33,345 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20c708165c0044cda75c25bdb0ebcd93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731708991038 2024-11-15T22:16:33,345 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5a986d2b0d54877881cc181eea1e2f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731708991248 2024-11-15T22:16:33,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T22:16:33,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T22:16:33,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:33,353 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-15T22:16:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:33,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/420067be199c487fa887d2475814d054 is 50, key is test_row_0/A:col10/1731708992389/Put/seqid=0 2024-11-15T22:16:33,357 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#C#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:33,358 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/dc09af8c450c4b3da08f6a450fcfa024 is 50, key is test_row_0/C:col10/1731708992372/Put/seqid=0 2024-11-15T22:16:33,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742087_1263 (size=12391) 2024-11-15T22:16:33,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742086_1262 (size=12151) 2024-11-15T22:16:33,369 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/dc09af8c450c4b3da08f6a450fcfa024 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/dc09af8c450c4b3da08f6a450fcfa024 2024-11-15T22:16:33,375 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/C of c203d77ebc02584fa5952f844fc15b8c into dc09af8c450c4b3da08f6a450fcfa024(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:33,375 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:33,375 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/C, priority=12, startTime=1731708993296; duration=0sec 2024-11-15T22:16:33,375 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:33,375 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:C 2024-11-15T22:16:33,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:33,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:33,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731708993460, max timestamp=1731708993467Num uniques 859; 2024-11-15T22:16:33,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:16:33,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731708993460, max timestamp=1731708993467Num uniques 859; 2024-11-15T22:16:33,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:16:33,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709053484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:33,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:33,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709053486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:33,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709053488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:33,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:33,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709053489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:33,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:33,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709053562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:33,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709053601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:33,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:33,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709053601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:33,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:33,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709053601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709053601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T22:16:33,734 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/77a2a3bb47854884aece3492f31ca8bd as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/77a2a3bb47854884aece3492f31ca8bd 2024-11-15T22:16:33,739 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/B of c203d77ebc02584fa5952f844fc15b8c into 77a2a3bb47854884aece3492f31ca8bd(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:33,739 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:33,739 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/B, priority=12, startTime=1731708993296; duration=0sec 2024-11-15T22:16:33,739 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:33,739 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:B 2024-11-15T22:16:33,770 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/420067be199c487fa887d2475814d054 2024-11-15T22:16:33,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/0a16a34d1caa4c3588d88ea902df30ad is 50, key is test_row_0/B:col10/1731708992389/Put/seqid=0 2024-11-15T22:16:33,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742088_1264 (size=12151) 2024-11-15T22:16:33,795 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/0a16a34d1caa4c3588d88ea902df30ad 2024-11-15T22:16:33,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/d266fd17fd85402eb0b6db33fe014841 is 50, key is test_row_0/C:col10/1731708992389/Put/seqid=0 2024-11-15T22:16:33,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709053811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:33,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709053812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:33,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709053812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709053812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:33,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742089_1265 (size=12151) 2024-11-15T22:16:33,823 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/d266fd17fd85402eb0b6db33fe014841 2024-11-15T22:16:33,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/420067be199c487fa887d2475814d054 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/420067be199c487fa887d2475814d054 2024-11-15T22:16:33,834 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/420067be199c487fa887d2475814d054, entries=150, sequenceid=186, filesize=11.9 K 2024-11-15T22:16:33,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/0a16a34d1caa4c3588d88ea902df30ad as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/0a16a34d1caa4c3588d88ea902df30ad 2024-11-15T22:16:33,838 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/0a16a34d1caa4c3588d88ea902df30ad, entries=150, sequenceid=186, filesize=11.9 K 2024-11-15T22:16:33,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/d266fd17fd85402eb0b6db33fe014841 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/d266fd17fd85402eb0b6db33fe014841 2024-11-15T22:16:33,844 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/d266fd17fd85402eb0b6db33fe014841, entries=150, sequenceid=186, filesize=11.9 K 2024-11-15T22:16:33,844 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=149.83 KB/153430 for c203d77ebc02584fa5952f844fc15b8c in 491ms, sequenceid=186, compaction requested=false 2024-11-15T22:16:33,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:33,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:33,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-15T22:16:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-15T22:16:33,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-15T22:16:33,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 809 msec 2024-11-15T22:16:33,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 814 msec 2024-11-15T22:16:34,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:34,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=156.54 KB heapSize=453.23 KB 2024-11-15T22:16:34,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:34,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:34,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:34,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:34,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:34,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:34,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/d3101a9f960b44759354da629d146819 is 50, key is test_row_0/A:col10/1731708994123/Put/seqid=0 2024-11-15T22:16:34,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709054125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709054126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:34,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742090_1266 (size=14541) 2024-11-15T22:16:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:34,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709054128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709054128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/d3101a9f960b44759354da629d146819 2024-11-15T22:16:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:34,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/d62298ba294a48e6a6b991745c8c5643 is 50, key is test_row_0/B:col10/1731708994123/Put/seqid=0 2024-11-15T22:16:34,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742091_1267 (size=12151) 2024-11-15T22:16:34,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T22:16:34,171 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:34,171 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:34,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-15T22:16:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T22:16:34,173 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:34,173 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:34,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:34,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:34,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709054242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709054242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:34,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709054245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:34,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709054242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:34,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T22:16:34,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T22:16:34,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:34,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:34,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:34,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:34,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:34,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709054451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:34,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709054452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:34,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709054452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709054452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:34,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T22:16:34,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:34,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:34,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:34,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:34,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:34,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:34,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T22:16:34,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/d62298ba294a48e6a6b991745c8c5643 2024-11-15T22:16:34,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/1df47c6d93eb47999b2a014e0ecc1926 is 50, key is test_row_0/C:col10/1731708994123/Put/seqid=0 2024-11-15T22:16:34,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742092_1268 (size=12151) 2024-11-15T22:16:34,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.56 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/1df47c6d93eb47999b2a014e0ecc1926 2024-11-15T22:16:34,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709054572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:34,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/d3101a9f960b44759354da629d146819 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/d3101a9f960b44759354da629d146819 2024-11-15T22:16:34,578 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in c203d77ebc02584fa5952f844fc15b8c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:16:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:34,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/d3101a9f960b44759354da629d146819, entries=200, sequenceid=223, filesize=14.2 K 2024-11-15T22:16:34,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/d62298ba294a48e6a6b991745c8c5643 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/d62298ba294a48e6a6b991745c8c5643 2024-11-15T22:16:34,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/d62298ba294a48e6a6b991745c8c5643, entries=150, sequenceid=223, filesize=11.9 K 2024-11-15T22:16:34,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/1df47c6d93eb47999b2a014e0ecc1926 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/1df47c6d93eb47999b2a014e0ecc1926 2024-11-15T22:16:34,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/1df47c6d93eb47999b2a014e0ecc1926, entries=150, sequenceid=223, filesize=11.9 K 2024-11-15T22:16:34,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~156.54 KB/160300, heapSize ~453.19 KB/464064, currentSize=20.13 KB/20610 for c203d77ebc02584fa5952f844fc15b8c in 487ms, sequenceid=223, compaction requested=true 2024-11-15T22:16:34,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:34,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:34,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:34,611 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:34,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:34,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:34,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:34,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:16:34,612 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:34,613 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:34,613 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/A is initiating minor compaction (all files) 2024-11-15T22:16:34,613 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/A in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:34,613 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/b0c6013924f24ea9bb06b9085a4ebf8b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/420067be199c487fa887d2475814d054, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/d3101a9f960b44759354da629d146819] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=38.2 K 2024-11-15T22:16:34,613 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0c6013924f24ea9bb06b9085a4ebf8b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731708991248 2024-11-15T22:16:34,613 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:34,613 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/B is initiating minor compaction (all files) 2024-11-15T22:16:34,614 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 420067be199c487fa887d2475814d054, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731708992389 2024-11-15T22:16:34,614 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/B in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:34,614 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/77a2a3bb47854884aece3492f31ca8bd, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/0a16a34d1caa4c3588d88ea902df30ad, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/d62298ba294a48e6a6b991745c8c5643] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=35.8 K 2024-11-15T22:16:34,614 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3101a9f960b44759354da629d146819, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731708993483 2024-11-15T22:16:34,615 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 77a2a3bb47854884aece3492f31ca8bd, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731708991248 2024-11-15T22:16:34,616 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a16a34d1caa4c3588d88ea902df30ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731708992389 2024-11-15T22:16:34,617 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d62298ba294a48e6a6b991745c8c5643, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731708993487 2024-11-15T22:16:34,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T22:16:34,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:34,632 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-15T22:16:34,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:34,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:34,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:34,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:34,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:34,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:34,638 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:34,638 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/f8d89f25d9c84427b748027d479771ad is 50, key is test_row_0/A:col10/1731708994123/Put/seqid=0 2024-11-15T22:16:34,644 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:34,645 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/ca5e0c1ba27c4c6ea7c9d16f02320701 is 50, key is test_row_0/B:col10/1731708994123/Put/seqid=0 2024-11-15T22:16:34,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/99ffae6fa94d42498befe377fc129e18 is 50, key is test_row_1/A:col10/1731708994126/Put/seqid=0 2024-11-15T22:16:34,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742094_1270 (size=12493) 2024-11-15T22:16:34,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742093_1269 (size=12493) 2024-11-15T22:16:34,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742095_1271 (size=9757) 2024-11-15T22:16:34,676 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/ca5e0c1ba27c4c6ea7c9d16f02320701 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/ca5e0c1ba27c4c6ea7c9d16f02320701 2024-11-15T22:16:34,688 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/B of c203d77ebc02584fa5952f844fc15b8c into ca5e0c1ba27c4c6ea7c9d16f02320701(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:34,688 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:34,688 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/B, priority=13, startTime=1731708994611; duration=0sec 2024-11-15T22:16:34,689 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:34,689 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:B 2024-11-15T22:16:34,689 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:34,693 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:34,693 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/C is initiating minor compaction (all files) 2024-11-15T22:16:34,693 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/C in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:34,693 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/dc09af8c450c4b3da08f6a450fcfa024, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/d266fd17fd85402eb0b6db33fe014841, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/1df47c6d93eb47999b2a014e0ecc1926] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=35.8 K 2024-11-15T22:16:34,694 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting dc09af8c450c4b3da08f6a450fcfa024, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731708991248 2024-11-15T22:16:34,694 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d266fd17fd85402eb0b6db33fe014841, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731708992389 2024-11-15T22:16:34,694 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 1df47c6d93eb47999b2a014e0ecc1926, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731708993487 2024-11-15T22:16:34,720 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#C#compaction#228 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:34,720 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/bb1dc6b78cc944a490cb5bdd334a92b9 is 50, key is test_row_0/C:col10/1731708994123/Put/seqid=0 2024-11-15T22:16:34,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742096_1272 (size=12493) 2024-11-15T22:16:34,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T22:16:34,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731708994795, max timestamp=1731708994831Num uniques 859; 2024-11-15T22:16:34,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T22:16:34,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731708994795, max timestamp=1731708994831Num uniques 859; 2024-11-15T22:16:34,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731708994795, max timestamp=1731708994831Num uniques 859; 2024-11-15T22:16:34,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T22:16:34,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T22:16:34,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709054906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:34,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:34,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:34,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731708994885, max timestamp=1731708994910Num uniques 1218; 2024-11-15T22:16:34,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731708994885, max timestamp=1731708994910Num uniques 1218; 2024-11-15T22:16:34,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=55778, new segment data size=7688, suffix heap size=154640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1218, new segment cells count=168 2024-11-15T22:16:34,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=55778, new segment data size=7688, suffix heap size=154640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1218, new segment cells count=168 2024-11-15T22:16:34,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731708994885, max timestamp=1731708994910Num uniques 1218; 2024-11-15T22:16:34,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=55778, new segment data size=6870, suffix heap size=154640,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1218, new segment cells count=150 2024-11-15T22:16:35,065 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/99ffae6fa94d42498befe377fc129e18 2024-11-15T22:16:35,069 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/f8d89f25d9c84427b748027d479771ad as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/f8d89f25d9c84427b748027d479771ad 2024-11-15T22:16:35,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/05659eaacd8b49f2a68622a6747deb84 is 50, key is test_row_1/B:col10/1731708994126/Put/seqid=0 2024-11-15T22:16:35,088 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/A of c203d77ebc02584fa5952f844fc15b8c into f8d89f25d9c84427b748027d479771ad(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:35,088 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:35,088 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/A, priority=13, startTime=1731708994611; duration=0sec 2024-11-15T22:16:35,088 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:35,088 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:A 2024-11-15T22:16:35,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742097_1273 (size=9757) 2024-11-15T22:16:35,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731708995056, max timestamp=1731708995103Num uniques 1027; 2024-11-15T22:16:35,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-15T22:16:35,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731708995056, max timestamp=1731708995103Num uniques 1027; 2024-11-15T22:16:35,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-15T22:16:35,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731708995056, max timestamp=1731708995103Num uniques 1009; 2024-11-15T22:16:35,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-11-15T22:16:35,168 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/bb1dc6b78cc944a490cb5bdd334a92b9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/bb1dc6b78cc944a490cb5bdd334a92b9 2024-11-15T22:16:35,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731708995141, max timestamp=1731708995168Num uniques 1036; 2024-11-15T22:16:35,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-15T22:16:35,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731708995141, max timestamp=1731708995168Num uniques 1036; 2024-11-15T22:16:35,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-15T22:16:35,176 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/C of c203d77ebc02584fa5952f844fc15b8c into bb1dc6b78cc944a490cb5bdd334a92b9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:35,176 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:35,176 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/C, priority=13, startTime=1731708994611; duration=0sec 2024-11-15T22:16:35,176 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:35,176 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:C 2024-11-15T22:16:35,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731708995141, max timestamp=1731708995168Num uniques 1036; 2024-11-15T22:16:35,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-11-15T22:16:35,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731708995254, max timestamp=1731708995266Num uniques 1095; 2024-11-15T22:16:35,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-15T22:16:35,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731708995254, max timestamp=1731708995266Num uniques 1095; 2024-11-15T22:16:35,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-15T22:16:35,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731708995254, max timestamp=1731708995266Num uniques 1009; 2024-11-15T22:16:35,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13510, suffix heap size=132904,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=295 2024-11-15T22:16:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T22:16:35,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731708995316, max timestamp=1731708995336Num uniques 1154; 2024-11-15T22:16:35,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-15T22:16:35,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731708995316, max timestamp=1731708995336Num uniques 1154; 2024-11-15T22:16:35,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-15T22:16:35,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731708995316, max timestamp=1731708995336Num uniques 1154; 2024-11-15T22:16:35,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-15T22:16:35,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731708995395, max timestamp=1731708995439Num uniques 1013; 2024-11-15T22:16:35,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-15T22:16:35,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731708995395, max timestamp=1731708995439Num uniques 1013; 2024-11-15T22:16:35,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-15T22:16:35,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731708995395, max timestamp=1731708995439Num uniques 1013; 2024-11-15T22:16:35,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-15T22:16:35,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731708995484, max timestamp=1731708995500Num uniques 1022; 2024-11-15T22:16:35,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-15T22:16:35,514 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/05659eaacd8b49f2a68622a6747deb84 2024-11-15T22:16:35,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731708995484, max timestamp=1731708995500Num uniques 1022; 2024-11-15T22:16:35,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-15T22:16:35,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731708995484, max timestamp=1731708995500Num uniques 1022; 2024-11-15T22:16:35,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-15T22:16:35,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/e90ca0bcdd314c628ca39e907c2c1a21 is 50, key is test_row_1/C:col10/1731708994126/Put/seqid=0 2024-11-15T22:16:35,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731708995560, max timestamp=1731708995579Num uniques 1031; 2024-11-15T22:16:35,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-15T22:16:35,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742098_1274 (size=9757) 2024-11-15T22:16:35,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731708995560, max timestamp=1731708995579Num uniques 1031; 2024-11-15T22:16:35,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-15T22:16:35,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731708995560, max timestamp=1731708995579Num uniques 1031; 2024-11-15T22:16:35,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-15T22:16:35,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731708995627, max timestamp=1731708995638Num uniques 1040; 2024-11-15T22:16:35,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731708995627, max timestamp=1731708995638Num uniques 1040; 2024-11-15T22:16:35,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-15T22:16:35,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-15T22:16:35,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731708995627, max timestamp=1731708995638Num uniques 1040; 2024-11-15T22:16:35,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-15T22:16:35,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731708995712, max timestamp=1731708995724Num uniques 1049; 2024-11-15T22:16:35,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=13694, suffix heap size=137064,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=299 2024-11-15T22:16:35,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1731708995712, max timestamp=1731708995727Num uniques 1049; 2024-11-15T22:16:35,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48034, new segment data size=9104, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-15T22:16:35,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1731708995712, max timestamp=1731708995727Num uniques 1049; 2024-11-15T22:16:35,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48034, new segment data size=9104, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-15T22:16:35,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731708995766, max timestamp=1731708995785Num uniques 1158; 2024-11-15T22:16:35,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731708995766, max timestamp=1731708995785Num uniques 1058; 2024-11-15T22:16:35,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-11-15T22:16:35,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-15T22:16:35,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731708995766, max timestamp=1731708995785Num uniques 1058; 2024-11-15T22:16:35,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-15T22:16:35,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1731708995825, max timestamp=1731708995841Num uniques 1017; 2024-11-15T22:16:35,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=12222, suffix heap size=133736,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=267 2024-11-15T22:16:35,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1731708995825, max timestamp=1731708995841Num uniques 1017; 2024-11-15T22:16:35,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=12222, suffix heap size=133736,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=267 2024-11-15T22:16:35,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731708995825, max timestamp=1731708995841Num uniques 1017; 2024-11-15T22:16:35,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=6870, suffix heap size=133736,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=150 2024-11-15T22:16:35,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731708995891, max timestamp=1731708995908Num uniques 1126; 2024-11-15T22:16:35,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=10346, suffix heap size=145072,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=226 2024-11-15T22:16:35,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731708995891, max timestamp=1731708995908Num uniques 1126; 2024-11-15T22:16:35,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=10346, suffix heap size=145072,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=226 2024-11-15T22:16:35,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731708995891, max timestamp=1731708995908Num uniques 1009; 2024-11-15T22:16:35,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-15T22:16:35,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731708995934, max timestamp=1731708995959Num uniques 1085; 2024-11-15T22:16:35,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-15T22:16:35,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731708995952, max timestamp=1731708995959Num uniques 1009; 2024-11-15T22:16:35,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-15T22:16:35,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731708995952, max timestamp=1731708995959Num uniques 1085; 2024-11-15T22:16:35,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=6870, suffix heap size=140808,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=150 2024-11-15T22:16:35,989 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/e90ca0bcdd314c628ca39e907c2c1a21 2024-11-15T22:16:36,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/99ffae6fa94d42498befe377fc129e18 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/99ffae6fa94d42498befe377fc129e18 2024-11-15T22:16:36,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731708995991, max timestamp=1731708996012Num uniques 1009; 2024-11-15T22:16:36,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731708995991, max timestamp=1731708996012Num uniques 1094; 2024-11-15T22:16:36,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=11174, suffix heap size=132904,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=244 2024-11-15T22:16:36,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-11-15T22:16:36,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731708995991, max timestamp=1731708996012Num uniques 1009; 2024-11-15T22:16:36,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=11174, suffix heap size=132904,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=244 2024-11-15T22:16:36,033 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/99ffae6fa94d42498befe377fc129e18, entries=100, sequenceid=229, filesize=9.5 K 2024-11-15T22:16:36,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/05659eaacd8b49f2a68622a6747deb84 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/05659eaacd8b49f2a68622a6747deb84 2024-11-15T22:16:36,042 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/05659eaacd8b49f2a68622a6747deb84, entries=100, sequenceid=229, filesize=9.5 K 2024-11-15T22:16:36,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/e90ca0bcdd314c628ca39e907c2c1a21 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e90ca0bcdd314c628ca39e907c2c1a21 2024-11-15T22:16:36,061 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e90ca0bcdd314c628ca39e907c2c1a21, entries=100, sequenceid=229, filesize=9.5 K 2024-11-15T22:16:36,065 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=93.93 KB/96180 for c203d77ebc02584fa5952f844fc15b8c in 1433ms, sequenceid=229, compaction requested=false 2024-11-15T22:16:36,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:36,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:36,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-15T22:16:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:36,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=100.63 KB heapSize=277.41 KB 2024-11-15T22:16:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-15T22:16:36,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:36,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:36,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:36,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:36,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:36,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:36,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-15T22:16:36,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9000 sec 2024-11-15T22:16:36,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/79bca7ae75df44bc8fdff49c1cfa7dfe is 50, key is test_row_0/A:col10/1731708996059/Put/seqid=0 2024-11-15T22:16:36,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.9180 sec 2024-11-15T22:16:36,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742099_1275 (size=17181) 2024-11-15T22:16:36,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/79bca7ae75df44bc8fdff49c1cfa7dfe 2024-11-15T22:16:36,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709056120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709056121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709056121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709056133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/78224a43fb6c4b4aa1243eefa49675de is 50, key is test_row_0/B:col10/1731708996059/Put/seqid=0 2024-11-15T22:16:36,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709056231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742100_1276 (size=12301) 2024-11-15T22:16:36,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709056242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709056242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709056247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T22:16:36,321 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:36,324 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-15T22:16:36,329 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:36,329 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:36,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T22:16:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T22:16:36,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709056441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709056454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709056454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709056461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T22:16:36,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:36,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:36,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:36,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:36,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:36,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709056604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,610 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in c203d77ebc02584fa5952f844fc15b8c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:16:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T22:16:36,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:36,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:36,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:36,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:36,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:36,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/78224a43fb6c4b4aa1243eefa49675de 2024-11-15T22:16:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T22:16:36,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/e59ae2e746ab4df2be4365324deaa1e6 is 50, key is test_row_0/C:col10/1731708996059/Put/seqid=0 2024-11-15T22:16:36,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742101_1277 (size=12301) 2024-11-15T22:16:36,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/e59ae2e746ab4df2be4365324deaa1e6 2024-11-15T22:16:36,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709056755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709056772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709056772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709056772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:36,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:36,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/79bca7ae75df44bc8fdff49c1cfa7dfe as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/79bca7ae75df44bc8fdff49c1cfa7dfe 2024-11-15T22:16:36,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T22:16:36,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/79bca7ae75df44bc8fdff49c1cfa7dfe, entries=250, sequenceid=520, filesize=16.8 K 2024-11-15T22:16:36,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:36,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:36,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:36,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:36,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:36,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/78224a43fb6c4b4aa1243eefa49675de as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/78224a43fb6c4b4aa1243eefa49675de 2024-11-15T22:16:36,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/78224a43fb6c4b4aa1243eefa49675de, entries=150, sequenceid=520, filesize=12.0 K 2024-11-15T22:16:36,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/e59ae2e746ab4df2be4365324deaa1e6 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e59ae2e746ab4df2be4365324deaa1e6 2024-11-15T22:16:36,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e59ae2e746ab4df2be4365324deaa1e6, entries=150, sequenceid=520, filesize=12.0 K 2024-11-15T22:16:36,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~277.36 KB/284016, currentSize=80.51 KB/82440 for c203d77ebc02584fa5952f844fc15b8c in 851ms, sequenceid=520, compaction requested=true 2024-11-15T22:16:36,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:36,919 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:36,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:36,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:36,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:36,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:36,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:36,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:16:36,920 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:36,933 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:36,933 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/A is initiating minor compaction (all files) 2024-11-15T22:16:36,933 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/A in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:36,933 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/f8d89f25d9c84427b748027d479771ad, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/99ffae6fa94d42498befe377fc129e18, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/79bca7ae75df44bc8fdff49c1cfa7dfe] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=38.5 K 2024-11-15T22:16:36,933 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:36,933 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/B is initiating minor compaction (all files) 2024-11-15T22:16:36,933 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/B in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:36,933 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/ca5e0c1ba27c4c6ea7c9d16f02320701, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/05659eaacd8b49f2a68622a6747deb84, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/78224a43fb6c4b4aa1243eefa49675de] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=33.7 K 2024-11-15T22:16:36,937 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca5e0c1ba27c4c6ea7c9d16f02320701, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731708993487 2024-11-15T22:16:36,937 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting f8d89f25d9c84427b748027d479771ad, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731708993487 2024-11-15T22:16:36,940 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05659eaacd8b49f2a68622a6747deb84, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731708994126 2024-11-15T22:16:36,941 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 99ffae6fa94d42498befe377fc129e18, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731708994126 2024-11-15T22:16:36,943 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78224a43fb6c4b4aa1243eefa49675de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1731708996054 2024-11-15T22:16:36,948 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 79bca7ae75df44bc8fdff49c1cfa7dfe, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1731708996041 2024-11-15T22:16:36,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T22:16:36,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T22:16:36,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:36,973 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T22:16:36,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:36,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:36,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:36,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:36,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:36,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/90291bc8f1b242229f39ecfbe84bfaf8 is 50, key is test_row_0/A:col10/1731708996121/Put/seqid=0 2024-11-15T22:16:37,022 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#B#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:37,023 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/31fba926c7294c47b2f6a540fb537fe5 is 50, key is test_row_0/B:col10/1731708996059/Put/seqid=0 2024-11-15T22:16:37,028 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#A#compaction#236 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:37,029 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/d2a3cba5250645178aca9eb66ecc968a is 50, key is test_row_0/A:col10/1731708996059/Put/seqid=0 2024-11-15T22:16:37,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742102_1278 (size=12301) 2024-11-15T22:16:37,057 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/90291bc8f1b242229f39ecfbe84bfaf8 2024-11-15T22:16:37,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742104_1280 (size=12745) 2024-11-15T22:16:37,075 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/d2a3cba5250645178aca9eb66ecc968a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/d2a3cba5250645178aca9eb66ecc968a 2024-11-15T22:16:37,092 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/A of c203d77ebc02584fa5952f844fc15b8c into d2a3cba5250645178aca9eb66ecc968a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:37,093 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:37,093 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/A, priority=13, startTime=1731708996919; duration=0sec 2024-11-15T22:16:37,093 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:37,093 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:A 2024-11-15T22:16:37,093 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:37,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742103_1279 (size=12745) 2024-11-15T22:16:37,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/cb7f6de659ab4c84a4be620cae0c00bb is 50, key is test_row_0/B:col10/1731708996121/Put/seqid=0 2024-11-15T22:16:37,103 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:37,104 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/C is initiating minor compaction (all files) 2024-11-15T22:16:37,104 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/C in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:37,104 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/bb1dc6b78cc944a490cb5bdd334a92b9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e90ca0bcdd314c628ca39e907c2c1a21, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e59ae2e746ab4df2be4365324deaa1e6] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=33.7 K 2024-11-15T22:16:37,112 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting bb1dc6b78cc944a490cb5bdd334a92b9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731708993487 2024-11-15T22:16:37,119 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting e90ca0bcdd314c628ca39e907c2c1a21, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731708994126 2024-11-15T22:16:37,128 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting e59ae2e746ab4df2be4365324deaa1e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1731708996054 2024-11-15T22:16:37,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742105_1281 (size=12301) 2024-11-15T22:16:37,150 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/31fba926c7294c47b2f6a540fb537fe5 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/31fba926c7294c47b2f6a540fb537fe5 2024-11-15T22:16:37,151 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/cb7f6de659ab4c84a4be620cae0c00bb 2024-11-15T22:16:37,168 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#C#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:37,168 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/8a84366139d447fd80688c09e177f98f is 50, key is test_row_0/C:col10/1731708996059/Put/seqid=0 2024-11-15T22:16:37,181 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/B of c203d77ebc02584fa5952f844fc15b8c into 31fba926c7294c47b2f6a540fb537fe5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:37,181 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:37,181 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/B, priority=13, startTime=1731708996920; duration=0sec 2024-11-15T22:16:37,181 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:37,181 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:B 2024-11-15T22:16:37,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742106_1282 (size=12745) 2024-11-15T22:16:37,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/e08c175a47314a6e9f8f620fa9a76aed is 50, key is test_row_0/C:col10/1731708996121/Put/seqid=0 2024-11-15T22:16:37,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742107_1283 (size=12301) 2024-11-15T22:16:37,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:37,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709057321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:37,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709057322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:37,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709057322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:37,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709057328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:37,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709057442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:37,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:37,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709057442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:37,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709057442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:37,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709057442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T22:16:37,631 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/8a84366139d447fd80688c09e177f98f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/8a84366139d447fd80688c09e177f98f 2024-11-15T22:16:37,650 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/C of c203d77ebc02584fa5952f844fc15b8c into 8a84366139d447fd80688c09e177f98f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:37,650 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:37,650 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/C, priority=13, startTime=1731708996920; duration=0sec 2024-11-15T22:16:37,651 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:37,651 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:C 2024-11-15T22:16:37,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709057653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:37,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709057653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:37,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709057653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:37,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:37,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709057654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:37,684 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/e08c175a47314a6e9f8f620fa9a76aed 2024-11-15T22:16:37,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/90291bc8f1b242229f39ecfbe84bfaf8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/90291bc8f1b242229f39ecfbe84bfaf8 2024-11-15T22:16:37,723 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/90291bc8f1b242229f39ecfbe84bfaf8, entries=150, sequenceid=535, filesize=12.0 K 2024-11-15T22:16:37,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/cb7f6de659ab4c84a4be620cae0c00bb as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/cb7f6de659ab4c84a4be620cae0c00bb 2024-11-15T22:16:37,750 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/cb7f6de659ab4c84a4be620cae0c00bb, entries=150, sequenceid=535, filesize=12.0 K 2024-11-15T22:16:37,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/e08c175a47314a6e9f8f620fa9a76aed as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e08c175a47314a6e9f8f620fa9a76aed 2024-11-15T22:16:37,760 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e08c175a47314a6e9f8f620fa9a76aed, entries=150, sequenceid=535, filesize=12.0 K 2024-11-15T22:16:37,761 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c203d77ebc02584fa5952f844fc15b8c in 788ms, sequenceid=535, compaction requested=false 2024-11-15T22:16:37,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:37,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:37,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-15T22:16:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-15T22:16:37,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-15T22:16:37,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4470 sec 2024-11-15T22:16:37,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.4720 sec 2024-11-15T22:16:37,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T22:16:37,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:37,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:37,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:37,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:37,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:37,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:37,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/589d772512cb48939558f33d25a2e38a is 50, key is test_row_0/A:col10/1731708997326/Put/seqid=0 2024-11-15T22:16:38,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742108_1284 (size=17181) 2024-11-15T22:16:38,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709058033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709058035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709058036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709058037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709058152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709058152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709058155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709058155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T22:16:38,501 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:38,502 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:38,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709058501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709058501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709058502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709058504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-15T22:16:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T22:16:38,526 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:38,531 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:38,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:38,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/589d772512cb48939558f33d25a2e38a 2024-11-15T22:16:38,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/53be8abf121a4693bcf77d33dc678aa7 is 50, key is test_row_0/B:col10/1731708997326/Put/seqid=0 2024-11-15T22:16:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T22:16:38,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742109_1285 (size=12301) 2024-11-15T22:16:38,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T22:16:38,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:38,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:38,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:38,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:38,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:38,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709058814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709058814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709058814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709058817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T22:16:38,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T22:16:38,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:38,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:38,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:38,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:38,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:39,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T22:16:39,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:39,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:39,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:39,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:39,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:39,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:39,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/53be8abf121a4693bcf77d33dc678aa7 2024-11-15T22:16:39,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/78e76992902f46c0a9d8af4224a8ef6b is 50, key is test_row_0/C:col10/1731708997326/Put/seqid=0 2024-11-15T22:16:39,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742110_1286 (size=12301) 2024-11-15T22:16:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T22:16:39,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T22:16:39,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:39,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:39,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:39,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:39,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:39,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T22:16:39,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:39,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:39,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:39,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:39,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:39,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709059332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:39,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:39,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:39,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709059333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:39,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:39,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709059333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:39,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709059332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:39,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:39,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:39,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:39,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T22:16:39,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:39,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:39,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:39,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:39,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:39,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/78e76992902f46c0a9d8af4224a8ef6b 2024-11-15T22:16:39,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/589d772512cb48939558f33d25a2e38a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/589d772512cb48939558f33d25a2e38a 2024-11-15T22:16:39,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/589d772512cb48939558f33d25a2e38a, entries=250, sequenceid=557, filesize=16.8 K 2024-11-15T22:16:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/53be8abf121a4693bcf77d33dc678aa7 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/53be8abf121a4693bcf77d33dc678aa7 2024-11-15T22:16:39,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/53be8abf121a4693bcf77d33dc678aa7, entries=150, sequenceid=557, filesize=12.0 K 2024-11-15T22:16:39,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/78e76992902f46c0a9d8af4224a8ef6b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/78e76992902f46c0a9d8af4224a8ef6b 2024-11-15T22:16:39,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/78e76992902f46c0a9d8af4224a8ef6b, entries=150, sequenceid=557, filesize=12.0 K 2024-11-15T22:16:39,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for c203d77ebc02584fa5952f844fc15b8c in 1683ms, sequenceid=557, compaction requested=true 2024-11-15T22:16:39,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:39,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:39,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:39,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:39,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:39,659 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:39,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:39,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:16:39,665 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T22:16:39,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T22:16:39,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:39,670 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T22:16:39,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:39,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:39,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:39,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:39,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:39,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:39,673 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:39,673 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/B is initiating minor compaction (all files) 2024-11-15T22:16:39,673 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/B in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:39,673 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/31fba926c7294c47b2f6a540fb537fe5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/cb7f6de659ab4c84a4be620cae0c00bb, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/53be8abf121a4693bcf77d33dc678aa7] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=36.5 K 2024-11-15T22:16:39,677 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42227 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:39,677 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 31fba926c7294c47b2f6a540fb537fe5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1731708996054 2024-11-15T22:16:39,677 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/A is initiating minor compaction (all files) 2024-11-15T22:16:39,677 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/A in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:39,678 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/d2a3cba5250645178aca9eb66ecc968a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/90291bc8f1b242229f39ecfbe84bfaf8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/589d772512cb48939558f33d25a2e38a] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=41.2 K 2024-11-15T22:16:39,679 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting cb7f6de659ab4c84a4be620cae0c00bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731708996111 2024-11-15T22:16:39,679 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2a3cba5250645178aca9eb66ecc968a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1731708996054 2024-11-15T22:16:39,681 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 53be8abf121a4693bcf77d33dc678aa7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1731708997326 2024-11-15T22:16:39,681 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90291bc8f1b242229f39ecfbe84bfaf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731708996111 2024-11-15T22:16:39,689 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 589d772512cb48939558f33d25a2e38a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1731708997315 2024-11-15T22:16:39,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/2f917f2a1cf64d888d24ff72d0c69105 is 50, key is test_row_0/A:col10/1731708998032/Put/seqid=0 2024-11-15T22:16:39,735 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:39,736 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/8c8a0cb38bab47298d029c574486bc42 is 50, key is test_row_0/B:col10/1731708997326/Put/seqid=0 2024-11-15T22:16:39,754 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#A#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:39,755 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/2858b84190d14c47885e95c8afcb6c58 is 50, key is test_row_0/A:col10/1731708997326/Put/seqid=0 2024-11-15T22:16:39,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742111_1287 (size=12301) 2024-11-15T22:16:39,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742113_1289 (size=12847) 2024-11-15T22:16:39,809 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/2858b84190d14c47885e95c8afcb6c58 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/2858b84190d14c47885e95c8afcb6c58 2024-11-15T22:16:39,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742112_1288 (size=12847) 2024-11-15T22:16:39,817 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/A of c203d77ebc02584fa5952f844fc15b8c into 2858b84190d14c47885e95c8afcb6c58(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:39,817 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:39,817 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/A, priority=13, startTime=1731708999659; duration=0sec 2024-11-15T22:16:39,817 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:39,817 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:A 2024-11-15T22:16:39,817 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:39,818 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:39,818 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/C is initiating minor compaction (all files) 2024-11-15T22:16:39,818 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/C in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:39,818 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/8a84366139d447fd80688c09e177f98f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e08c175a47314a6e9f8f620fa9a76aed, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/78e76992902f46c0a9d8af4224a8ef6b] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=36.5 K 2024-11-15T22:16:39,819 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a84366139d447fd80688c09e177f98f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1731708996054 2024-11-15T22:16:39,820 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting e08c175a47314a6e9f8f620fa9a76aed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731708996111 2024-11-15T22:16:39,821 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78e76992902f46c0a9d8af4224a8ef6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1731708997326 2024-11-15T22:16:39,828 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/8c8a0cb38bab47298d029c574486bc42 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/8c8a0cb38bab47298d029c574486bc42 2024-11-15T22:16:39,844 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/B of c203d77ebc02584fa5952f844fc15b8c into 8c8a0cb38bab47298d029c574486bc42(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:39,844 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:39,844 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/B, priority=13, startTime=1731708999659; duration=0sec 2024-11-15T22:16:39,844 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:39,845 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:B 2024-11-15T22:16:39,847 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#C#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:39,847 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/8113a5c35a6547a7b53f57de5b3c038b is 50, key is test_row_0/C:col10/1731708997326/Put/seqid=0 2024-11-15T22:16:39,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742114_1290 (size=12847) 2024-11-15T22:16:39,876 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/8113a5c35a6547a7b53f57de5b3c038b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/8113a5c35a6547a7b53f57de5b3c038b 2024-11-15T22:16:39,887 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/C of c203d77ebc02584fa5952f844fc15b8c into 8113a5c35a6547a7b53f57de5b3c038b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:39,887 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:39,887 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/C, priority=13, startTime=1731708999659; duration=0sec 2024-11-15T22:16:39,888 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:39,888 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:C 2024-11-15T22:16:40,185 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/2f917f2a1cf64d888d24ff72d0c69105 2024-11-15T22:16:40,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/aef894b5aa0c4f82a039d6ebb6f6fcf2 is 50, key is test_row_0/B:col10/1731708998032/Put/seqid=0 2024-11-15T22:16:40,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742115_1291 (size=12301) 2024-11-15T22:16:40,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:40,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709060396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:40,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:40,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709060402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:40,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:40,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:40,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709060406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:40,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:40,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709060408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:40,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:40,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709060512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:40,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:40,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709060516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:40,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709060521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709060522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:40,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:40,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:40,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57100 deadline: 1731709060622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:40,629 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in c203d77ebc02584fa5952f844fc15b8c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:16:40,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T22:16:40,688 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/aef894b5aa0c4f82a039d6ebb6f6fcf2 2024-11-15T22:16:40,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709060721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:40,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:40,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709060732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:40,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/f109f3ca8d894b44ac4b6a09e9963293 is 50, key is test_row_0/C:col10/1731708998032/Put/seqid=0 2024-11-15T22:16:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:40,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709060732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:40,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709060734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:40,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742116_1292 (size=12301) 2024-11-15T22:16:41,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709061042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:41,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709061053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709061054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:41,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:41,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709061053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:41,188 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/f109f3ca8d894b44ac4b6a09e9963293 2024-11-15T22:16:41,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/2f917f2a1cf64d888d24ff72d0c69105 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/2f917f2a1cf64d888d24ff72d0c69105 2024-11-15T22:16:41,224 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/2f917f2a1cf64d888d24ff72d0c69105, entries=150, sequenceid=571, filesize=12.0 K 2024-11-15T22:16:41,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/aef894b5aa0c4f82a039d6ebb6f6fcf2 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/aef894b5aa0c4f82a039d6ebb6f6fcf2 2024-11-15T22:16:41,237 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/aef894b5aa0c4f82a039d6ebb6f6fcf2, entries=150, sequenceid=571, filesize=12.0 K 2024-11-15T22:16:41,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/f109f3ca8d894b44ac4b6a09e9963293 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f109f3ca8d894b44ac4b6a09e9963293 2024-11-15T22:16:41,247 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f109f3ca8d894b44ac4b6a09e9963293, entries=150, sequenceid=571, filesize=12.0 K 2024-11-15T22:16:41,248 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for c203d77ebc02584fa5952f844fc15b8c in 1579ms, sequenceid=571, compaction requested=false 2024-11-15T22:16:41,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:41,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:41,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-15T22:16:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-15T22:16:41,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-15T22:16:41,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7200 sec 2024-11-15T22:16:41,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.7560 sec 2024-11-15T22:16:41,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T22:16:41,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:41,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:41,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:41,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/c2b3486fe9dd40e6acaa282a3ba6892a is 50, key is test_row_0/A:col10/1731709000403/Put/seqid=0 2024-11-15T22:16:41,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709061605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709061607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709061612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709061613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:41,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742117_1293 (size=12301) 2024-11-15T22:16:41,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709061721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:41,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709061721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:41,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709061732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:41,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709061736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:41,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709061942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709061942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:41,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709061942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:41,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709061951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:42,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/c2b3486fe9dd40e6acaa282a3ba6892a 2024-11-15T22:16:42,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/f7bb9b2f12ce4a8c8e33e01604874f47 is 50, key is test_row_0/B:col10/1731709000403/Put/seqid=0 2024-11-15T22:16:42,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742118_1294 (size=12301) 2024-11-15T22:16:42,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709062256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:42,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709062256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:42,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709062256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:42,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:42,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709062264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:42,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/f7bb9b2f12ce4a8c8e33e01604874f47 2024-11-15T22:16:42,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/48765620777e4e829eff7d54bf4f4db1 is 50, key is test_row_0/C:col10/1731709000403/Put/seqid=0 2024-11-15T22:16:42,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742119_1295 (size=12301) 2024-11-15T22:16:42,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/48765620777e4e829eff7d54bf4f4db1 2024-11-15T22:16:42,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/c2b3486fe9dd40e6acaa282a3ba6892a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/c2b3486fe9dd40e6acaa282a3ba6892a 2024-11-15T22:16:42,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/c2b3486fe9dd40e6acaa282a3ba6892a, entries=150, sequenceid=594, filesize=12.0 K 2024-11-15T22:16:42,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/f7bb9b2f12ce4a8c8e33e01604874f47 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/f7bb9b2f12ce4a8c8e33e01604874f47 2024-11-15T22:16:42,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/f7bb9b2f12ce4a8c8e33e01604874f47, entries=150, sequenceid=594, filesize=12.0 K 2024-11-15T22:16:42,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/48765620777e4e829eff7d54bf4f4db1 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/48765620777e4e829eff7d54bf4f4db1 2024-11-15T22:16:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T22:16:42,693 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:42,693 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:42,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/48765620777e4e829eff7d54bf4f4db1, entries=150, sequenceid=594, filesize=12.0 K 2024-11-15T22:16:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-15T22:16:42,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for c203d77ebc02584fa5952f844fc15b8c in 1130ms, sequenceid=594, compaction requested=true 2024-11-15T22:16:42,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T22:16:42,701 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:42,701 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:42,702 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:42,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:42,702 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:42,702 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/A is initiating minor compaction (all files) 2024-11-15T22:16:42,702 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/A in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:42,703 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/2858b84190d14c47885e95c8afcb6c58, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/2f917f2a1cf64d888d24ff72d0c69105, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/c2b3486fe9dd40e6acaa282a3ba6892a] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=36.6 K 2024-11-15T22:16:42,704 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2858b84190d14c47885e95c8afcb6c58, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1731708997326 2024-11-15T22:16:42,705 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f917f2a1cf64d888d24ff72d0c69105, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731708998016 2024-11-15T22:16:42,705 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2b3486fe9dd40e6acaa282a3ba6892a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1731709000364 2024-11-15T22:16:42,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:42,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:42,719 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:42,720 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:42,720 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/B is initiating minor compaction (all files) 2024-11-15T22:16:42,721 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/B in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:42,721 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/8c8a0cb38bab47298d029c574486bc42, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/aef894b5aa0c4f82a039d6ebb6f6fcf2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/f7bb9b2f12ce4a8c8e33e01604874f47] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=36.6 K 2024-11-15T22:16:42,722 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c8a0cb38bab47298d029c574486bc42, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1731708997326 2024-11-15T22:16:42,722 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting aef894b5aa0c4f82a039d6ebb6f6fcf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731708998016 2024-11-15T22:16:42,722 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting f7bb9b2f12ce4a8c8e33e01604874f47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1731709000364 2024-11-15T22:16:42,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:42,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:42,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:42,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:42,730 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#A#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:42,731 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/1dc44638034a40a9997e5f3c0457f41c is 50, key is test_row_0/A:col10/1731709000403/Put/seqid=0 2024-11-15T22:16:42,736 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#B#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:42,736 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/f7454908702d48a1ae77430516aacf2d is 50, key is test_row_0/B:col10/1731709000403/Put/seqid=0 2024-11-15T22:16:42,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T22:16:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:42,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742120_1296 (size=12949) 2024-11-15T22:16:42,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:42,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:42,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:42,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:42,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:42,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:42,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742121_1297 (size=12949) 2024-11-15T22:16:42,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/99bd77821eb4490385d99065295e9513 is 50, key is test_row_0/A:col10/1731709001590/Put/seqid=0 2024-11-15T22:16:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T22:16:42,813 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/f7454908702d48a1ae77430516aacf2d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/f7454908702d48a1ae77430516aacf2d 2024-11-15T22:16:42,831 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/B of c203d77ebc02584fa5952f844fc15b8c into f7454908702d48a1ae77430516aacf2d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:42,831 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:42,831 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/B, priority=13, startTime=1731709002719; duration=0sec 2024-11-15T22:16:42,831 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:42,831 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:B 2024-11-15T22:16:42,831 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:42,834 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:42,834 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/C is initiating minor compaction (all files) 2024-11-15T22:16:42,834 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/C in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:42,834 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/8113a5c35a6547a7b53f57de5b3c038b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f109f3ca8d894b44ac4b6a09e9963293, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/48765620777e4e829eff7d54bf4f4db1] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=36.6 K 2024-11-15T22:16:42,836 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 8113a5c35a6547a7b53f57de5b3c038b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1731708997326 2024-11-15T22:16:42,838 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting f109f3ca8d894b44ac4b6a09e9963293, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731708998016 2024-11-15T22:16:42,840 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 48765620777e4e829eff7d54bf4f4db1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1731709000364 2024-11-15T22:16:42,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742122_1298 (size=14741) 2024-11-15T22:16:42,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T22:16:42,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:42,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:42,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:42,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:42,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709062857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:42,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:42,891 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#C#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:42,892 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/dbfe254ae46d427996a524df5d27d852 is 50, key is test_row_0/C:col10/1731709000403/Put/seqid=0 2024-11-15T22:16:42,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709062876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:42,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709062887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:42,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709062895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:42,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742123_1299 (size=12949) 2024-11-15T22:16:43,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709062995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709063002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T22:16:43,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:43,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:43,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:43,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:43,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T22:16:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:43,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709063022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709063031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T22:16:43,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:43,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:43,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:43,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:43,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:43,197 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/1dc44638034a40a9997e5f3c0457f41c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/1dc44638034a40a9997e5f3c0457f41c 2024-11-15T22:16:43,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709063211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,215 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/A of c203d77ebc02584fa5952f844fc15b8c into 1dc44638034a40a9997e5f3c0457f41c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:43,215 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:43,215 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/A, priority=13, startTime=1731709002700; duration=0sec 2024-11-15T22:16:43,215 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:43,215 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:A 2024-11-15T22:16:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709063213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709063231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/99bd77821eb4490385d99065295e9513 2024-11-15T22:16:43,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709063252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/97f02538da60430f88ccafdc58deaca5 is 50, key is test_row_0/B:col10/1731709001590/Put/seqid=0 2024-11-15T22:16:43,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742124_1300 (size=12301) 2024-11-15T22:16:43,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/97f02538da60430f88ccafdc58deaca5 2024-11-15T22:16:43,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T22:16:43,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T22:16:43,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:43,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:43,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:43,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:43,346 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/dbfe254ae46d427996a524df5d27d852 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/dbfe254ae46d427996a524df5d27d852 2024-11-15T22:16:43,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:43,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/d3cf6d79617a41329d8cbd1e3c73ba72 is 50, key is test_row_0/C:col10/1731709001590/Put/seqid=0 2024-11-15T22:16:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:43,356 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/C of c203d77ebc02584fa5952f844fc15b8c into dbfe254ae46d427996a524df5d27d852(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:43,356 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:43,356 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/C, priority=13, startTime=1731709002728; duration=0sec 2024-11-15T22:16:43,356 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:43,356 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:C 2024-11-15T22:16:43,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742125_1301 (size=12301) 2024-11-15T22:16:43,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/d3cf6d79617a41329d8cbd1e3c73ba72 2024-11-15T22:16:43,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/99bd77821eb4490385d99065295e9513 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/99bd77821eb4490385d99065295e9513 2024-11-15T22:16:43,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/99bd77821eb4490385d99065295e9513, entries=200, sequenceid=609, filesize=14.4 K 2024-11-15T22:16:43,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/97f02538da60430f88ccafdc58deaca5 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/97f02538da60430f88ccafdc58deaca5 2024-11-15T22:16:43,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/97f02538da60430f88ccafdc58deaca5, entries=150, sequenceid=609, filesize=12.0 K 2024-11-15T22:16:43,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/d3cf6d79617a41329d8cbd1e3c73ba72 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/d3cf6d79617a41329d8cbd1e3c73ba72 2024-11-15T22:16:43,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/d3cf6d79617a41329d8cbd1e3c73ba72, entries=150, sequenceid=609, filesize=12.0 K 2024-11-15T22:16:43,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c203d77ebc02584fa5952f844fc15b8c in 726ms, sequenceid=609, compaction requested=false 2024-11-15T22:16:43,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:43,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T22:16:43,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:43,508 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T22:16:43,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:43,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:43,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:43,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:43,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:43,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:43,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/80a62e6f66a9425e8c7fe92fd3421025 is 50, key is test_row_0/A:col10/1731709002872/Put/seqid=0 2024-11-15T22:16:43,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:43,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742126_1302 (size=12301) 2024-11-15T22:16:43,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709063638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709063644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709063654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709063650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709063762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709063763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709063773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709063782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T22:16:43,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709063984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:43,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709063993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:44,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709064002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:44,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709064003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:44,025 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=630 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/80a62e6f66a9425e8c7fe92fd3421025 2024-11-15T22:16:44,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/53b3e36905624504b0493549269616b7 is 50, key is test_row_0/B:col10/1731709002872/Put/seqid=0 2024-11-15T22:16:44,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742127_1303 (size=12301) 2024-11-15T22:16:44,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709064292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:44,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:44,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709064313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:44,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709064314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:44,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709064314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:44,511 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=630 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/53b3e36905624504b0493549269616b7 2024-11-15T22:16:44,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/ea620fdf390546afadc069f48b284ffc is 50, key is test_row_0/C:col10/1731709002872/Put/seqid=0 2024-11-15T22:16:44,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742128_1304 (size=12301) 2024-11-15T22:16:44,585 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=630 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/ea620fdf390546afadc069f48b284ffc 2024-11-15T22:16:44,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/80a62e6f66a9425e8c7fe92fd3421025 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/80a62e6f66a9425e8c7fe92fd3421025 2024-11-15T22:16:44,702 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/80a62e6f66a9425e8c7fe92fd3421025, entries=150, sequenceid=630, filesize=12.0 K 2024-11-15T22:16:44,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/53b3e36905624504b0493549269616b7 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/53b3e36905624504b0493549269616b7 2024-11-15T22:16:44,741 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/53b3e36905624504b0493549269616b7, entries=150, sequenceid=630, filesize=12.0 K 2024-11-15T22:16:44,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/ea620fdf390546afadc069f48b284ffc as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/ea620fdf390546afadc069f48b284ffc 2024-11-15T22:16:44,789 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/ea620fdf390546afadc069f48b284ffc, entries=150, sequenceid=630, filesize=12.0 K 2024-11-15T22:16:44,796 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for c203d77ebc02584fa5952f844fc15b8c in 1288ms, sequenceid=630, compaction requested=true 2024-11-15T22:16:44,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:44,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:44,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-15T22:16:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-15T22:16:44,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T22:16:44,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:44,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:44,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:44,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:44,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:44,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:44,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:44,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/808225bf955743fba1d25017001effcf is 50, key is test_row_0/A:col10/1731709004808/Put/seqid=0 2024-11-15T22:16:44,828 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-15T22:16:44,828 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1020 sec 2024-11-15T22:16:44,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.1400 sec 2024-11-15T22:16:44,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742129_1305 (size=14741) 2024-11-15T22:16:44,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T22:16:44,871 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:44,872 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-15T22:16:44,878 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:44,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T22:16:44,882 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:44,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:44,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:44,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709064885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:44,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:44,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:44,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:44,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709064895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:44,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709064892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:44,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:44,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709064899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:44,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T22:16:45,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709065012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:45,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:45,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709065012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:45,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:45,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709065014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:45,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:45,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709065014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:45,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T22:16:45,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:45,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:45,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:45,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:45,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:45,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:45,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T22:16:45,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:45,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:45,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:45,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:45,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:45,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T22:16:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:45,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709065221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:45,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:45,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709065221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:45,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:45,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709065232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709065232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:45,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/808225bf955743fba1d25017001effcf 2024-11-15T22:16:45,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/9dbfe80b1a154b7b87db0deea6e3b09f is 50, key is test_row_0/B:col10/1731709004808/Put/seqid=0 2024-11-15T22:16:45,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742130_1306 (size=12301) 2024-11-15T22:16:45,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/9dbfe80b1a154b7b87db0deea6e3b09f 2024-11-15T22:16:45,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T22:16:45,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:45,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:45,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:45,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:45,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:45,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/ed17363b8206472d936d6d98e0c29ba6 is 50, key is test_row_0/C:col10/1731709004808/Put/seqid=0 2024-11-15T22:16:45,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742131_1307 (size=12301) 2024-11-15T22:16:45,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/ed17363b8206472d936d6d98e0c29ba6 2024-11-15T22:16:45,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/808225bf955743fba1d25017001effcf as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/808225bf955743fba1d25017001effcf 2024-11-15T22:16:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T22:16:45,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709065532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:45,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709065534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:45,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/808225bf955743fba1d25017001effcf, entries=200, sequenceid=646, filesize=14.4 K 2024-11-15T22:16:45,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T22:16:45,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/9dbfe80b1a154b7b87db0deea6e3b09f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/9dbfe80b1a154b7b87db0deea6e3b09f 2024-11-15T22:16:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:45,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:45,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:45,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709065543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:45,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709065543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:45,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/9dbfe80b1a154b7b87db0deea6e3b09f, entries=150, sequenceid=646, filesize=12.0 K 2024-11-15T22:16:45,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/ed17363b8206472d936d6d98e0c29ba6 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/ed17363b8206472d936d6d98e0c29ba6 2024-11-15T22:16:45,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T22:16:45,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:45,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:45,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:45,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:45,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:45,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/ed17363b8206472d936d6d98e0c29ba6, entries=150, sequenceid=646, filesize=12.0 K 2024-11-15T22:16:45,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for c203d77ebc02584fa5952f844fc15b8c in 910ms, sequenceid=646, compaction requested=true 2024-11-15T22:16:45,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:45,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:45,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:45,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:45,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:16:45,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:45,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T22:16:45,721 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:16:45,724 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:16:45,735 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:16:45,735 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/A is initiating minor compaction (all files) 2024-11-15T22:16:45,736 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/A in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:45,736 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/1dc44638034a40a9997e5f3c0457f41c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/99bd77821eb4490385d99065295e9513, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/80a62e6f66a9425e8c7fe92fd3421025, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/808225bf955743fba1d25017001effcf] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=53.4 K 2024-11-15T22:16:45,736 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dc44638034a40a9997e5f3c0457f41c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1731709000364 2024-11-15T22:16:45,742 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 99bd77821eb4490385d99065295e9513, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1731709001590 2024-11-15T22:16:45,744 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 80a62e6f66a9425e8c7fe92fd3421025, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=630, earliestPutTs=1731709002838 2024-11-15T22:16:45,744 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 808225bf955743fba1d25017001effcf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1731709003616 2024-11-15T22:16:45,748 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:16:45,748 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/C is initiating minor compaction (all files) 2024-11-15T22:16:45,748 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/C in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:45,748 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/dbfe254ae46d427996a524df5d27d852, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/d3cf6d79617a41329d8cbd1e3c73ba72, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/ea620fdf390546afadc069f48b284ffc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/ed17363b8206472d936d6d98e0c29ba6] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=48.7 K 2024-11-15T22:16:45,765 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbfe254ae46d427996a524df5d27d852, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1731709000364 2024-11-15T22:16:45,773 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3cf6d79617a41329d8cbd1e3c73ba72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1731709001590 2024-11-15T22:16:45,784 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea620fdf390546afadc069f48b284ffc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=630, earliestPutTs=1731709002838 2024-11-15T22:16:45,785 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed17363b8206472d936d6d98e0c29ba6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1731709003616 2024-11-15T22:16:45,831 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#A#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:45,832 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/4e13b9af14da4978abff44d909fdf2f0 is 50, key is test_row_0/A:col10/1731709004808/Put/seqid=0 2024-11-15T22:16:45,839 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#C#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:45,840 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/09f601052c0b442d9868f6ce527b4244 is 50, key is test_row_0/C:col10/1731709004808/Put/seqid=0 2024-11-15T22:16:45,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T22:16:45,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:45,868 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T22:16:45,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:45,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:45,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:45,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:45,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:45,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:45,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742132_1308 (size=13085) 2024-11-15T22:16:45,902 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/4e13b9af14da4978abff44d909fdf2f0 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/4e13b9af14da4978abff44d909fdf2f0 2024-11-15T22:16:45,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742133_1309 (size=13085) 2024-11-15T22:16:45,917 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/A of c203d77ebc02584fa5952f844fc15b8c into 4e13b9af14da4978abff44d909fdf2f0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:45,917 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:45,917 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/A, priority=12, startTime=1731709005721; duration=0sec 2024-11-15T22:16:45,917 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:45,917 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:A 2024-11-15T22:16:45,917 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:16:45,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/4f6b1776b2684599ad9a6fc0d17e2e88 is 50, key is test_row_0/A:col10/1731709004847/Put/seqid=0 2024-11-15T22:16:45,925 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:16:45,925 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/B is initiating minor compaction (all files) 2024-11-15T22:16:45,925 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/B in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:45,925 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/f7454908702d48a1ae77430516aacf2d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/97f02538da60430f88ccafdc58deaca5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/53b3e36905624504b0493549269616b7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/9dbfe80b1a154b7b87db0deea6e3b09f] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=48.7 K 2024-11-15T22:16:45,925 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting f7454908702d48a1ae77430516aacf2d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1731709000364 2024-11-15T22:16:45,926 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 97f02538da60430f88ccafdc58deaca5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1731709001590 2024-11-15T22:16:45,928 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 53b3e36905624504b0493549269616b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=630, earliestPutTs=1731709002838 2024-11-15T22:16:45,930 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dbfe80b1a154b7b87db0deea6e3b09f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1731709003616 2024-11-15T22:16:45,934 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/09f601052c0b442d9868f6ce527b4244 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/09f601052c0b442d9868f6ce527b4244 2024-11-15T22:16:45,954 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/C of c203d77ebc02584fa5952f844fc15b8c into 09f601052c0b442d9868f6ce527b4244(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:45,954 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:45,954 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/C, priority=12, startTime=1731709005721; duration=0sec 2024-11-15T22:16:45,954 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:45,958 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:C 2024-11-15T22:16:45,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742134_1310 (size=12301) 2024-11-15T22:16:45,971 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#B#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:45,971 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/5b99c793748b4cc081b89e23d9365086 is 50, key is test_row_0/B:col10/1731709004808/Put/seqid=0 2024-11-15T22:16:46,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742135_1311 (size=13085) 2024-11-15T22:16:46,044 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/5b99c793748b4cc081b89e23d9365086 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/5b99c793748b4cc081b89e23d9365086 2024-11-15T22:16:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T22:16:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:46,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. as already flushing 2024-11-15T22:16:46,073 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/B of c203d77ebc02584fa5952f844fc15b8c into 5b99c793748b4cc081b89e23d9365086(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:46,073 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:46,073 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/B, priority=12, startTime=1731709005721; duration=0sec 2024-11-15T22:16:46,073 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:46,073 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:B 2024-11-15T22:16:46,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709066137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709066141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709066160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709066165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709066272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709066272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709066300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709066302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,361 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=663 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/4f6b1776b2684599ad9a6fc0d17e2e88 2024-11-15T22:16:46,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/71e3a4f443b740648e10118f993f237b is 50, key is test_row_0/B:col10/1731709004847/Put/seqid=0 2024-11-15T22:16:46,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742136_1312 (size=12301) 2024-11-15T22:16:46,440 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=663 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/71e3a4f443b740648e10118f993f237b 2024-11-15T22:16:46,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/9f168678badf44fab5e0b34bd73b7839 is 50, key is test_row_0/C:col10/1731709004847/Put/seqid=0 2024-11-15T22:16:46,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709066492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709066492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709066524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742137_1313 (size=12301) 2024-11-15T22:16:46,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709066534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,769 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-11-15T22:16:46,769 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:46,769 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:46,769 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:46,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:46,771 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-11-15T22:16:46,771 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:46,771 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:46,771 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:46,774 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-11-15T22:16:46,774 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:46,774 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:46,774 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:46,774 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:46,776 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-15T22:16:46,776 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:46,776 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:46,776 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:46,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:46,778 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-11-15T22:16:46,778 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:46,778 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:46,778 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:46,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:46,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:46,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57164 deadline: 1731709066801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57140 deadline: 1731709066811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57098 deadline: 1731709066844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57124 deadline: 1731709066852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:46,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:46,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:16:46,936 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=663 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/9f168678badf44fab5e0b34bd73b7839 2024-11-15T22:16:46,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/4f6b1776b2684599ad9a6fc0d17e2e88 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/4f6b1776b2684599ad9a6fc0d17e2e88 2024-11-15T22:16:46,943 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/4f6b1776b2684599ad9a6fc0d17e2e88, entries=150, sequenceid=663, filesize=12.0 K 2024-11-15T22:16:46,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/71e3a4f443b740648e10118f993f237b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/71e3a4f443b740648e10118f993f237b 2024-11-15T22:16:46,947 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/71e3a4f443b740648e10118f993f237b, entries=150, sequenceid=663, filesize=12.0 K 2024-11-15T22:16:46,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/9f168678badf44fab5e0b34bd73b7839 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/9f168678badf44fab5e0b34bd73b7839 2024-11-15T22:16:46,954 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/9f168678badf44fab5e0b34bd73b7839, entries=150, sequenceid=663, filesize=12.0 K 2024-11-15T22:16:46,955 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c203d77ebc02584fa5952f844fc15b8c in 1087ms, sequenceid=663, compaction requested=false 2024-11-15T22:16:46,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:46,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:46,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-15T22:16:46,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-15T22:16:46,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-15T22:16:46,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0740 sec 2024-11-15T22:16:46,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.0860 sec 2024-11-15T22:16:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T22:16:47,052 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:47,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T22:16:47,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:47,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:47,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:47,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:47,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:47,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:47,328 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-11-15T22:16:47,328 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:47,328 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:47,328 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:47,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:47,331 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-15T22:16:47,331 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:47,331 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:47,331 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:47,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:47,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/de085f1586174888a099f58bf199fda8 is 50, key is test_row_0/A:col10/1731709007316/Put/seqid=0 2024-11-15T22:16:47,367 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-11-15T22:16:47,367 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:47,367 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:47,367 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:47,368 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:47,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742138_1314 (size=12301) 2024-11-15T22:16:47,376 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-11-15T22:16:47,376 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:47,376 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:47,376 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:47,377 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:47,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/de085f1586174888a099f58bf199fda8 2024-11-15T22:16:47,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/fd678a0a3ee34fe891809fd4b6223945 is 50, key is test_row_0/B:col10/1731709007316/Put/seqid=0 2024-11-15T22:16:47,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742139_1315 (size=12301) 2024-11-15T22:16:47,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/fd678a0a3ee34fe891809fd4b6223945 2024-11-15T22:16:47,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/e4352c31a4604aae8b7e2358ec6b5d6f is 50, key is test_row_0/C:col10/1731709007316/Put/seqid=0 2024-11-15T22:16:47,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742140_1316 (size=12301) 2024-11-15T22:16:47,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/e4352c31a4604aae8b7e2358ec6b5d6f 2024-11-15T22:16:47,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/de085f1586174888a099f58bf199fda8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/de085f1586174888a099f58bf199fda8 2024-11-15T22:16:47,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/de085f1586174888a099f58bf199fda8, entries=150, sequenceid=682, filesize=12.0 K 2024-11-15T22:16:47,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/fd678a0a3ee34fe891809fd4b6223945 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/fd678a0a3ee34fe891809fd4b6223945 2024-11-15T22:16:47,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/fd678a0a3ee34fe891809fd4b6223945, entries=150, sequenceid=682, filesize=12.0 K 2024-11-15T22:16:47,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/e4352c31a4604aae8b7e2358ec6b5d6f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e4352c31a4604aae8b7e2358ec6b5d6f 2024-11-15T22:16:47,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e4352c31a4604aae8b7e2358ec6b5d6f, entries=150, sequenceid=682, filesize=12.0 K 2024-11-15T22:16:47,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=20.13 KB/20610 for c203d77ebc02584fa5952f844fc15b8c in 647ms, sequenceid=682, compaction requested=true 2024-11-15T22:16:47,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:47,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:47,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:47,972 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:47,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:47,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:47,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c203d77ebc02584fa5952f844fc15b8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:47,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:16:47,973 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:47,973 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/A is initiating minor compaction (all files) 2024-11-15T22:16:47,973 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/A in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:47,973 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/4e13b9af14da4978abff44d909fdf2f0, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/4f6b1776b2684599ad9a6fc0d17e2e88, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/de085f1586174888a099f58bf199fda8] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=36.8 K 2024-11-15T22:16:47,974 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e13b9af14da4978abff44d909fdf2f0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1731709003616 2024-11-15T22:16:47,974 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f6b1776b2684599ad9a6fc0d17e2e88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=663, earliestPutTs=1731709004847 2024-11-15T22:16:47,975 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting de085f1586174888a099f58bf199fda8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1731709006147 2024-11-15T22:16:47,976 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:47,988 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:47,988 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/B is initiating minor compaction (all files) 2024-11-15T22:16:47,988 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/B in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:47,988 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/5b99c793748b4cc081b89e23d9365086, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/71e3a4f443b740648e10118f993f237b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/fd678a0a3ee34fe891809fd4b6223945] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=36.8 K 2024-11-15T22:16:47,990 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#A#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:47,991 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/f6d4c21635354d968f31089cfd6c5ef5 is 50, key is test_row_0/A:col10/1731709007316/Put/seqid=0 2024-11-15T22:16:47,992 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b99c793748b4cc081b89e23d9365086, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1731709003616 2024-11-15T22:16:47,992 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 71e3a4f443b740648e10118f993f237b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=663, earliestPutTs=1731709004847 2024-11-15T22:16:47,996 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting fd678a0a3ee34fe891809fd4b6223945, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1731709006147 2024-11-15T22:16:48,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742141_1317 (size=13187) 2024-11-15T22:16:48,051 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#B#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:48,052 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/88ed3699981b4e6496473423768b41ef is 50, key is test_row_0/B:col10/1731709007316/Put/seqid=0 2024-11-15T22:16:48,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742142_1318 (size=13187) 2024-11-15T22:16:48,440 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/f6d4c21635354d968f31089cfd6c5ef5 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/f6d4c21635354d968f31089cfd6c5ef5 2024-11-15T22:16:48,469 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/A of c203d77ebc02584fa5952f844fc15b8c into f6d4c21635354d968f31089cfd6c5ef5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:48,469 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:48,469 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/A, priority=13, startTime=1731709007972; duration=0sec 2024-11-15T22:16:48,469 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:48,470 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:A 2024-11-15T22:16:48,470 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:48,476 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:48,476 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): c203d77ebc02584fa5952f844fc15b8c/C is initiating minor compaction (all files) 2024-11-15T22:16:48,476 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c203d77ebc02584fa5952f844fc15b8c/C in TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:48,476 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/09f601052c0b442d9868f6ce527b4244, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/9f168678badf44fab5e0b34bd73b7839, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e4352c31a4604aae8b7e2358ec6b5d6f] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp, totalSize=36.8 K 2024-11-15T22:16:48,478 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09f601052c0b442d9868f6ce527b4244, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1731709003616 2024-11-15T22:16:48,479 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f168678badf44fab5e0b34bd73b7839, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=663, earliestPutTs=1731709004847 2024-11-15T22:16:48,487 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4352c31a4604aae8b7e2358ec6b5d6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1731709006147 2024-11-15T22:16:48,487 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/88ed3699981b4e6496473423768b41ef as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/88ed3699981b4e6496473423768b41ef 2024-11-15T22:16:48,504 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/B of c203d77ebc02584fa5952f844fc15b8c into 88ed3699981b4e6496473423768b41ef(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:48,504 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:48,504 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/B, priority=13, startTime=1731709007972; duration=0sec 2024-11-15T22:16:48,504 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:48,504 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:B 2024-11-15T22:16:48,519 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c203d77ebc02584fa5952f844fc15b8c#C#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:48,519 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/000a190b425c4d85832c2707e7000b8d is 50, key is test_row_0/C:col10/1731709007316/Put/seqid=0 2024-11-15T22:16:48,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742143_1319 (size=13187) 2024-11-15T22:16:48,556 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/000a190b425c4d85832c2707e7000b8d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/000a190b425c4d85832c2707e7000b8d 2024-11-15T22:16:48,577 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c203d77ebc02584fa5952f844fc15b8c/C of c203d77ebc02584fa5952f844fc15b8c into 000a190b425c4d85832c2707e7000b8d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:48,577 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c203d77ebc02584fa5952f844fc15b8c: 2024-11-15T22:16:48,577 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., storeName=c203d77ebc02584fa5952f844fc15b8c/C, priority=13, startTime=1731709007972; duration=0sec 2024-11-15T22:16:48,577 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:48,577 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c203d77ebc02584fa5952f844fc15b8c:C 2024-11-15T22:16:50,673 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-11-15T22:16:50,673 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:16:50,673 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:50,675 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:50,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T22:16:50,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-15T22:16:50,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-15T22:16:50,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 147 2024-11-15T22:16:50,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-11-15T22:16:50,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 147 2024-11-15T22:16:50,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T22:16:50,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1859 2024-11-15T22:16:50,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1832 2024-11-15T22:16:50,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1873 2024-11-15T22:16:50,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1874 2024-11-15T22:16:50,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1819 2024-11-15T22:16:50,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:50,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T22:16:50,676 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:16:50,676 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:16:50,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:50,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:50,677 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:16:50,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T22:16:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T22:16:50,685 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731709010685"}]},"ts":"1731709010685"} 2024-11-15T22:16:50,689 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T22:16:50,689 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T22:16:50,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:16:50,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c203d77ebc02584fa5952f844fc15b8c, UNASSIGN}] 2024-11-15T22:16:50,694 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c203d77ebc02584fa5952f844fc15b8c, UNASSIGN 2024-11-15T22:16:50,695 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=c203d77ebc02584fa5952f844fc15b8c, regionState=CLOSING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:16:50,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c203d77ebc02584fa5952f844fc15b8c, UNASSIGN because future has completed 2024-11-15T22:16:50,697 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:16:50,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:16:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T22:16:50,853 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(122): Close c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:50,853 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:16:50,853 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1722): Closing c203d77ebc02584fa5952f844fc15b8c, disabling compactions & flushes 2024-11-15T22:16:50,853 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:50,853 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:50,853 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. after waiting 0 ms 2024-11-15T22:16:50,853 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:50,853 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(2902): Flushing c203d77ebc02584fa5952f844fc15b8c 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-15T22:16:50,854 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=A 2024-11-15T22:16:50,854 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:50,854 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=B 2024-11-15T22:16:50,854 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:50,854 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c203d77ebc02584fa5952f844fc15b8c, store=C 2024-11-15T22:16:50,854 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:50,886 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/2544369341ef47028dbf87e5766ced1f is 50, key is test_row_0/A:col10/1731709007360/Put/seqid=0 2024-11-15T22:16:50,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742144_1320 (size=9857) 2024-11-15T22:16:50,933 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/2544369341ef47028dbf87e5766ced1f 2024-11-15T22:16:50,982 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/27e6696ea80846e0bc707c8407ff8c9c is 50, key is test_row_0/B:col10/1731709007360/Put/seqid=0 2024-11-15T22:16:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T22:16:51,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742145_1321 (size=9857) 2024-11-15T22:16:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T22:16:51,435 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/27e6696ea80846e0bc707c8407ff8c9c 2024-11-15T22:16:51,474 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/f2c150f7f01c4ce4a4eb7307493d38bc is 50, key is test_row_0/C:col10/1731709007360/Put/seqid=0 2024-11-15T22:16:51,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742146_1322 (size=9857) 2024-11-15T22:16:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T22:16:51,921 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/f2c150f7f01c4ce4a4eb7307493d38bc 2024-11-15T22:16:52,107 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/A/2544369341ef47028dbf87e5766ced1f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/2544369341ef47028dbf87e5766ced1f 2024-11-15T22:16:52,112 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/2544369341ef47028dbf87e5766ced1f, entries=100, sequenceid=692, filesize=9.6 K 2024-11-15T22:16:52,112 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/B/27e6696ea80846e0bc707c8407ff8c9c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/27e6696ea80846e0bc707c8407ff8c9c 2024-11-15T22:16:52,118 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/27e6696ea80846e0bc707c8407ff8c9c, entries=100, sequenceid=692, filesize=9.6 K 2024-11-15T22:16:52,123 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/.tmp/C/f2c150f7f01c4ce4a4eb7307493d38bc as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f2c150f7f01c4ce4a4eb7307493d38bc 2024-11-15T22:16:52,127 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f2c150f7f01c4ce4a4eb7307493d38bc, entries=100, sequenceid=692, filesize=9.6 K 2024-11-15T22:16:52,128 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c203d77ebc02584fa5952f844fc15b8c in 1274ms, sequenceid=692, compaction requested=false 2024-11-15T22:16:52,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/1497b0f6a81d4d9ba08bff7e0edf7228, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/36ab7b2464de436e929c67606deb5606, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/21b44306ab5a409d99ae8b86dfa81249, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0e7f2a42e42f41448a00295e9e8be52c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0b9e0cddf8b24f4d847d9bf74589df3f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0f939796012b42709005fcb95cfa39ee, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/f82dfd346d5849e48bfc3a58094cd31c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/b0c6013924f24ea9bb06b9085a4ebf8b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/420067be199c487fa887d2475814d054, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/d3101a9f960b44759354da629d146819, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/f8d89f25d9c84427b748027d479771ad, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/99ffae6fa94d42498befe377fc129e18, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/79bca7ae75df44bc8fdff49c1cfa7dfe, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/d2a3cba5250645178aca9eb66ecc968a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/90291bc8f1b242229f39ecfbe84bfaf8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/589d772512cb48939558f33d25a2e38a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/2858b84190d14c47885e95c8afcb6c58, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/2f917f2a1cf64d888d24ff72d0c69105, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/1dc44638034a40a9997e5f3c0457f41c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/c2b3486fe9dd40e6acaa282a3ba6892a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/99bd77821eb4490385d99065295e9513, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/80a62e6f66a9425e8c7fe92fd3421025, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/808225bf955743fba1d25017001effcf, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/4e13b9af14da4978abff44d909fdf2f0, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/4f6b1776b2684599ad9a6fc0d17e2e88, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/de085f1586174888a099f58bf199fda8] to archive 2024-11-15T22:16:52,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:16:52,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/1497b0f6a81d4d9ba08bff7e0edf7228 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/1497b0f6a81d4d9ba08bff7e0edf7228 2024-11-15T22:16:52,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/36ab7b2464de436e929c67606deb5606 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/36ab7b2464de436e929c67606deb5606 2024-11-15T22:16:52,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/21b44306ab5a409d99ae8b86dfa81249 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/21b44306ab5a409d99ae8b86dfa81249 2024-11-15T22:16:52,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0e7f2a42e42f41448a00295e9e8be52c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0e7f2a42e42f41448a00295e9e8be52c 2024-11-15T22:16:52,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0b9e0cddf8b24f4d847d9bf74589df3f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0b9e0cddf8b24f4d847d9bf74589df3f 2024-11-15T22:16:52,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0f939796012b42709005fcb95cfa39ee to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/0f939796012b42709005fcb95cfa39ee 2024-11-15T22:16:52,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/f82dfd346d5849e48bfc3a58094cd31c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/f82dfd346d5849e48bfc3a58094cd31c 2024-11-15T22:16:52,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/b0c6013924f24ea9bb06b9085a4ebf8b to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/b0c6013924f24ea9bb06b9085a4ebf8b 2024-11-15T22:16:52,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/420067be199c487fa887d2475814d054 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/420067be199c487fa887d2475814d054 2024-11-15T22:16:52,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/d3101a9f960b44759354da629d146819 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/d3101a9f960b44759354da629d146819 2024-11-15T22:16:52,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/f8d89f25d9c84427b748027d479771ad to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/f8d89f25d9c84427b748027d479771ad 2024-11-15T22:16:52,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/99ffae6fa94d42498befe377fc129e18 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/99ffae6fa94d42498befe377fc129e18 2024-11-15T22:16:52,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/79bca7ae75df44bc8fdff49c1cfa7dfe to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/79bca7ae75df44bc8fdff49c1cfa7dfe 2024-11-15T22:16:52,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/d2a3cba5250645178aca9eb66ecc968a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/d2a3cba5250645178aca9eb66ecc968a 2024-11-15T22:16:52,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/90291bc8f1b242229f39ecfbe84bfaf8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/90291bc8f1b242229f39ecfbe84bfaf8 2024-11-15T22:16:52,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/589d772512cb48939558f33d25a2e38a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/589d772512cb48939558f33d25a2e38a 2024-11-15T22:16:52,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/2858b84190d14c47885e95c8afcb6c58 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/2858b84190d14c47885e95c8afcb6c58 2024-11-15T22:16:52,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/2f917f2a1cf64d888d24ff72d0c69105 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/2f917f2a1cf64d888d24ff72d0c69105 2024-11-15T22:16:52,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/1dc44638034a40a9997e5f3c0457f41c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/1dc44638034a40a9997e5f3c0457f41c 2024-11-15T22:16:52,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/c2b3486fe9dd40e6acaa282a3ba6892a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/c2b3486fe9dd40e6acaa282a3ba6892a 2024-11-15T22:16:52,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/99bd77821eb4490385d99065295e9513 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/99bd77821eb4490385d99065295e9513 2024-11-15T22:16:52,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/80a62e6f66a9425e8c7fe92fd3421025 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/80a62e6f66a9425e8c7fe92fd3421025 2024-11-15T22:16:52,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/808225bf955743fba1d25017001effcf to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/808225bf955743fba1d25017001effcf 2024-11-15T22:16:52,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/4e13b9af14da4978abff44d909fdf2f0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/4e13b9af14da4978abff44d909fdf2f0 2024-11-15T22:16:52,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/4f6b1776b2684599ad9a6fc0d17e2e88 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/4f6b1776b2684599ad9a6fc0d17e2e88 2024-11-15T22:16:52,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/de085f1586174888a099f58bf199fda8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/de085f1586174888a099f58bf199fda8 2024-11-15T22:16:52,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/ea9cec487be24ac0b76ba9bc7056de6b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/bbe92d11a8094071b4b5298492630402, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/dda22b8703e746f7bb252e23ae6a232e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/16d3db7fb8814ccd9e95b2fe5106c67a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/1ec6987f32fc4f65b9017c120538a8bc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/b307ff3f90df425aaeb06fa3fa869ca8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/77a2a3bb47854884aece3492f31ca8bd, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/8014abf52a98485db86b5e3af8935fc1, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/0a16a34d1caa4c3588d88ea902df30ad, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/ca5e0c1ba27c4c6ea7c9d16f02320701, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/d62298ba294a48e6a6b991745c8c5643, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/05659eaacd8b49f2a68622a6747deb84, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/31fba926c7294c47b2f6a540fb537fe5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/78224a43fb6c4b4aa1243eefa49675de, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/cb7f6de659ab4c84a4be620cae0c00bb, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/8c8a0cb38bab47298d029c574486bc42, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/53be8abf121a4693bcf77d33dc678aa7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/aef894b5aa0c4f82a039d6ebb6f6fcf2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/f7454908702d48a1ae77430516aacf2d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/f7bb9b2f12ce4a8c8e33e01604874f47, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/97f02538da60430f88ccafdc58deaca5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/53b3e36905624504b0493549269616b7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/5b99c793748b4cc081b89e23d9365086, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/9dbfe80b1a154b7b87db0deea6e3b09f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/71e3a4f443b740648e10118f993f237b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/fd678a0a3ee34fe891809fd4b6223945] to archive 2024-11-15T22:16:52,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:16:52,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/ea9cec487be24ac0b76ba9bc7056de6b to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/ea9cec487be24ac0b76ba9bc7056de6b 2024-11-15T22:16:52,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/bbe92d11a8094071b4b5298492630402 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/bbe92d11a8094071b4b5298492630402 2024-11-15T22:16:52,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/dda22b8703e746f7bb252e23ae6a232e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/dda22b8703e746f7bb252e23ae6a232e 2024-11-15T22:16:52,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/16d3db7fb8814ccd9e95b2fe5106c67a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/16d3db7fb8814ccd9e95b2fe5106c67a 2024-11-15T22:16:52,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/1ec6987f32fc4f65b9017c120538a8bc to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/1ec6987f32fc4f65b9017c120538a8bc 2024-11-15T22:16:52,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/b307ff3f90df425aaeb06fa3fa869ca8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/b307ff3f90df425aaeb06fa3fa869ca8 2024-11-15T22:16:52,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/77a2a3bb47854884aece3492f31ca8bd to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/77a2a3bb47854884aece3492f31ca8bd 2024-11-15T22:16:52,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/8014abf52a98485db86b5e3af8935fc1 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/8014abf52a98485db86b5e3af8935fc1 2024-11-15T22:16:52,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/0a16a34d1caa4c3588d88ea902df30ad to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/0a16a34d1caa4c3588d88ea902df30ad 2024-11-15T22:16:52,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/ca5e0c1ba27c4c6ea7c9d16f02320701 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/ca5e0c1ba27c4c6ea7c9d16f02320701 2024-11-15T22:16:52,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/d62298ba294a48e6a6b991745c8c5643 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/d62298ba294a48e6a6b991745c8c5643 2024-11-15T22:16:52,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/05659eaacd8b49f2a68622a6747deb84 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/05659eaacd8b49f2a68622a6747deb84 2024-11-15T22:16:52,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/31fba926c7294c47b2f6a540fb537fe5 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/31fba926c7294c47b2f6a540fb537fe5 2024-11-15T22:16:52,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/78224a43fb6c4b4aa1243eefa49675de to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/78224a43fb6c4b4aa1243eefa49675de 2024-11-15T22:16:52,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/cb7f6de659ab4c84a4be620cae0c00bb to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/cb7f6de659ab4c84a4be620cae0c00bb 2024-11-15T22:16:52,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/8c8a0cb38bab47298d029c574486bc42 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/8c8a0cb38bab47298d029c574486bc42 2024-11-15T22:16:52,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/53be8abf121a4693bcf77d33dc678aa7 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/53be8abf121a4693bcf77d33dc678aa7 2024-11-15T22:16:52,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/aef894b5aa0c4f82a039d6ebb6f6fcf2 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/aef894b5aa0c4f82a039d6ebb6f6fcf2 2024-11-15T22:16:52,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/f7454908702d48a1ae77430516aacf2d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/f7454908702d48a1ae77430516aacf2d 2024-11-15T22:16:52,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/f7bb9b2f12ce4a8c8e33e01604874f47 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/f7bb9b2f12ce4a8c8e33e01604874f47 2024-11-15T22:16:52,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/97f02538da60430f88ccafdc58deaca5 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/97f02538da60430f88ccafdc58deaca5 2024-11-15T22:16:52,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/53b3e36905624504b0493549269616b7 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/53b3e36905624504b0493549269616b7 2024-11-15T22:16:52,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/5b99c793748b4cc081b89e23d9365086 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/5b99c793748b4cc081b89e23d9365086 2024-11-15T22:16:52,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/9dbfe80b1a154b7b87db0deea6e3b09f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/9dbfe80b1a154b7b87db0deea6e3b09f 2024-11-15T22:16:52,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/71e3a4f443b740648e10118f993f237b to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/71e3a4f443b740648e10118f993f237b 2024-11-15T22:16:52,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/fd678a0a3ee34fe891809fd4b6223945 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/fd678a0a3ee34fe891809fd4b6223945 2024-11-15T22:16:52,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/544b331b497b427ead789d7e36df814c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f783ba49100642bfb050d091f40e1a38, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/5898c078c91345e89bf1055ac3b1c3ae, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e7e05e1fd0934713b7d5353bf6db7414, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/1191eba62a1f4d01968db8c913187470, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/20c708165c0044cda75c25bdb0ebcd93, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/dc09af8c450c4b3da08f6a450fcfa024, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f5a986d2b0d54877881cc181eea1e2f6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/d266fd17fd85402eb0b6db33fe014841, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/bb1dc6b78cc944a490cb5bdd334a92b9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/1df47c6d93eb47999b2a014e0ecc1926, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e90ca0bcdd314c628ca39e907c2c1a21, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/8a84366139d447fd80688c09e177f98f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e59ae2e746ab4df2be4365324deaa1e6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e08c175a47314a6e9f8f620fa9a76aed, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/8113a5c35a6547a7b53f57de5b3c038b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/78e76992902f46c0a9d8af4224a8ef6b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f109f3ca8d894b44ac4b6a09e9963293, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/dbfe254ae46d427996a524df5d27d852, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/48765620777e4e829eff7d54bf4f4db1, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/d3cf6d79617a41329d8cbd1e3c73ba72, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/ea620fdf390546afadc069f48b284ffc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/09f601052c0b442d9868f6ce527b4244, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/ed17363b8206472d936d6d98e0c29ba6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/9f168678badf44fab5e0b34bd73b7839, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e4352c31a4604aae8b7e2358ec6b5d6f] to archive 2024-11-15T22:16:52,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:16:52,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/544b331b497b427ead789d7e36df814c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/544b331b497b427ead789d7e36df814c 2024-11-15T22:16:52,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f783ba49100642bfb050d091f40e1a38 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f783ba49100642bfb050d091f40e1a38 2024-11-15T22:16:52,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/5898c078c91345e89bf1055ac3b1c3ae to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/5898c078c91345e89bf1055ac3b1c3ae 2024-11-15T22:16:52,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e7e05e1fd0934713b7d5353bf6db7414 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e7e05e1fd0934713b7d5353bf6db7414 2024-11-15T22:16:52,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/1191eba62a1f4d01968db8c913187470 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/1191eba62a1f4d01968db8c913187470 2024-11-15T22:16:52,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/20c708165c0044cda75c25bdb0ebcd93 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/20c708165c0044cda75c25bdb0ebcd93 2024-11-15T22:16:52,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/dc09af8c450c4b3da08f6a450fcfa024 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/dc09af8c450c4b3da08f6a450fcfa024 2024-11-15T22:16:52,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f5a986d2b0d54877881cc181eea1e2f6 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f5a986d2b0d54877881cc181eea1e2f6 2024-11-15T22:16:52,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/d266fd17fd85402eb0b6db33fe014841 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/d266fd17fd85402eb0b6db33fe014841 2024-11-15T22:16:52,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/bb1dc6b78cc944a490cb5bdd334a92b9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/bb1dc6b78cc944a490cb5bdd334a92b9 2024-11-15T22:16:52,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/1df47c6d93eb47999b2a014e0ecc1926 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/1df47c6d93eb47999b2a014e0ecc1926 2024-11-15T22:16:52,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e90ca0bcdd314c628ca39e907c2c1a21 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e90ca0bcdd314c628ca39e907c2c1a21 2024-11-15T22:16:52,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/8a84366139d447fd80688c09e177f98f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/8a84366139d447fd80688c09e177f98f 2024-11-15T22:16:52,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e59ae2e746ab4df2be4365324deaa1e6 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e59ae2e746ab4df2be4365324deaa1e6 2024-11-15T22:16:52,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e08c175a47314a6e9f8f620fa9a76aed to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e08c175a47314a6e9f8f620fa9a76aed 2024-11-15T22:16:52,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/8113a5c35a6547a7b53f57de5b3c038b to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/8113a5c35a6547a7b53f57de5b3c038b 2024-11-15T22:16:52,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/78e76992902f46c0a9d8af4224a8ef6b to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/78e76992902f46c0a9d8af4224a8ef6b 2024-11-15T22:16:52,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f109f3ca8d894b44ac4b6a09e9963293 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f109f3ca8d894b44ac4b6a09e9963293 2024-11-15T22:16:52,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/dbfe254ae46d427996a524df5d27d852 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/dbfe254ae46d427996a524df5d27d852 2024-11-15T22:16:52,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/48765620777e4e829eff7d54bf4f4db1 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/48765620777e4e829eff7d54bf4f4db1 2024-11-15T22:16:52,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/d3cf6d79617a41329d8cbd1e3c73ba72 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/d3cf6d79617a41329d8cbd1e3c73ba72 2024-11-15T22:16:52,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/ea620fdf390546afadc069f48b284ffc to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/ea620fdf390546afadc069f48b284ffc 2024-11-15T22:16:52,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/09f601052c0b442d9868f6ce527b4244 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/09f601052c0b442d9868f6ce527b4244 2024-11-15T22:16:52,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/ed17363b8206472d936d6d98e0c29ba6 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/ed17363b8206472d936d6d98e0c29ba6 2024-11-15T22:16:52,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/9f168678badf44fab5e0b34bd73b7839 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/9f168678badf44fab5e0b34bd73b7839 2024-11-15T22:16:52,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e4352c31a4604aae8b7e2358ec6b5d6f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/e4352c31a4604aae8b7e2358ec6b5d6f 2024-11-15T22:16:52,529 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/recovered.edits/695.seqid, newMaxSeqId=695, maxSeqId=1 2024-11-15T22:16:52,530 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. 2024-11-15T22:16:52,530 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1676): Region close journal for c203d77ebc02584fa5952f844fc15b8c: Waiting for close lock at 1731709010853Running coprocessor pre-close hooks at 1731709010853Disabling compacts and flushes for region at 1731709010853Disabling writes for close at 1731709010853Obtaining lock to block concurrent updates at 1731709010853Preparing flush snapshotting stores in c203d77ebc02584fa5952f844fc15b8c at 1731709010853Finished memstore snapshotting TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731709010854 (+1 ms)Flushing stores of TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c. at 1731709010862 (+8 ms)Flushing c203d77ebc02584fa5952f844fc15b8c/A: creating writer at 1731709010862Flushing c203d77ebc02584fa5952f844fc15b8c/A: appending metadata at 1731709010885 (+23 ms)Flushing c203d77ebc02584fa5952f844fc15b8c/A: closing flushed file at 1731709010885Flushing c203d77ebc02584fa5952f844fc15b8c/B: creating writer at 1731709010961 (+76 ms)Flushing c203d77ebc02584fa5952f844fc15b8c/B: appending metadata at 1731709010981 (+20 ms)Flushing c203d77ebc02584fa5952f844fc15b8c/B: closing flushed file at 1731709010981Flushing c203d77ebc02584fa5952f844fc15b8c/C: creating writer at 1731709011466 (+485 ms)Flushing c203d77ebc02584fa5952f844fc15b8c/C: appending metadata at 1731709011473 (+7 ms)Flushing c203d77ebc02584fa5952f844fc15b8c/C: closing flushed file at 1731709011473Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32d47733: reopening flushed file at 1731709012106 (+633 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@729e8144: reopening flushed file at 1731709012112 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@641947c6: reopening flushed file at 1731709012118 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c203d77ebc02584fa5952f844fc15b8c in 1274ms, sequenceid=692, compaction requested=false at 1731709012128 (+10 ms)Writing region close event to WAL at 1731709012514 (+386 ms)Running coprocessor post-close hooks at 1731709012530 (+16 ms)Closed at 1731709012530 2024-11-15T22:16:52,538 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(157): Closed c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:52,542 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=c203d77ebc02584fa5952f844fc15b8c, regionState=CLOSED 2024-11-15T22:16:52,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:16:52,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-15T22:16:52,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseRegionProcedure c203d77ebc02584fa5952f844fc15b8c, server=e611192d6313,38779,1731708927034 in 1.8550 sec 2024-11-15T22:16:52,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-11-15T22:16:52,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c203d77ebc02584fa5952f844fc15b8c, UNASSIGN in 1.8750 sec 2024-11-15T22:16:52,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-15T22:16:52,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8930 sec 2024-11-15T22:16:52,589 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731709012589"}]},"ts":"1731709012589"} 2024-11-15T22:16:52,594 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T22:16:52,594 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T22:16:52,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9170 sec 2024-11-15T22:16:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T22:16:52,842 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:52,843 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T22:16:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:52,846 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T22:16:52,857 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:52,882 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:52,898 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/recovered.edits] 2024-11-15T22:16:52,900 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/2544369341ef47028dbf87e5766ced1f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/2544369341ef47028dbf87e5766ced1f 2024-11-15T22:16:52,902 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/f6d4c21635354d968f31089cfd6c5ef5 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/A/f6d4c21635354d968f31089cfd6c5ef5 2024-11-15T22:16:52,912 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/27e6696ea80846e0bc707c8407ff8c9c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/27e6696ea80846e0bc707c8407ff8c9c 2024-11-15T22:16:52,914 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/88ed3699981b4e6496473423768b41ef to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/B/88ed3699981b4e6496473423768b41ef 2024-11-15T22:16:52,919 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/000a190b425c4d85832c2707e7000b8d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/000a190b425c4d85832c2707e7000b8d 2024-11-15T22:16:52,932 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f2c150f7f01c4ce4a4eb7307493d38bc to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/C/f2c150f7f01c4ce4a4eb7307493d38bc 2024-11-15T22:16:52,938 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/recovered.edits/695.seqid to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c/recovered.edits/695.seqid 2024-11-15T22:16:52,939 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/c203d77ebc02584fa5952f844fc15b8c 2024-11-15T22:16:52,939 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T22:16:52,942 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:52,946 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T22:16:52,953 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T22:16:52,954 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:52,954 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T22:16:52,954 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731709012954"}]},"ts":"9223372036854775807"} 2024-11-15T22:16:52,956 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T22:16:52,956 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c203d77ebc02584fa5952f844fc15b8c, NAME => 'TestAcidGuarantees,,1731708985482.c203d77ebc02584fa5952f844fc15b8c.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T22:16:52,956 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T22:16:52,956 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731709012956"}]},"ts":"9223372036854775807"} 2024-11-15T22:16:52,959 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T22:16:52,960 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T22:16:52,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 117 msec 2024-11-15T22:16:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T22:16:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T22:16:53,172 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:53,183 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 252), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1297 (was 1053) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=257 (was 246) - AvailableMemoryMB LEAK? - 2024-11-15T22:16:53,196 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=1297, ProcessCount=11, AvailableMemoryMB=256 2024-11-15T22:16:53,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:16:53,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:16:53,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:53,201 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T22:16:53,201 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:53,201 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 79 2024-11-15T22:16:53,202 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T22:16:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-15T22:16:53,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742147_1323 (size=1061) 2024-11-15T22:16:53,229 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e 2024-11-15T22:16:53,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742148_1324 (size=53) 2024-11-15T22:16:53,250 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:16:53,250 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 94761791c1172512c89bb88a26d6fda0, disabling compactions & flushes 2024-11-15T22:16:53,250 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:53,250 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:53,250 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. after waiting 0 ms 2024-11-15T22:16:53,250 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:53,250 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:53,250 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 94761791c1172512c89bb88a26d6fda0: Waiting for close lock at 1731709013250Disabling compacts and flushes for region at 1731709013250Disabling writes for close at 1731709013250Writing region close event to WAL at 1731709013250Closed at 1731709013250 2024-11-15T22:16:53,251 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T22:16:53,252 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731709013251"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731709013251"}]},"ts":"1731709013251"} 2024-11-15T22:16:53,253 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T22:16:53,254 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T22:16:53,254 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731709013254"}]},"ts":"1731709013254"} 2024-11-15T22:16:53,257 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T22:16:53,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94761791c1172512c89bb88a26d6fda0, ASSIGN}] 2024-11-15T22:16:53,264 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94761791c1172512c89bb88a26d6fda0, ASSIGN 2024-11-15T22:16:53,268 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94761791c1172512c89bb88a26d6fda0, ASSIGN; state=OFFLINE, location=e611192d6313,38779,1731708927034; forceNewPlan=false, retain=false 2024-11-15T22:16:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-15T22:16:53,419 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=94761791c1172512c89bb88a26d6fda0, regionState=OPENING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:16:53,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94761791c1172512c89bb88a26d6fda0, ASSIGN because future has completed 2024-11-15T22:16:53,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:16:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-15T22:16:53,577 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:53,577 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:16:53,578 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:53,578 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:16:53,578 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:53,578 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:53,590 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:53,594 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:16:53,594 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94761791c1172512c89bb88a26d6fda0 columnFamilyName A 2024-11-15T22:16:53,594 DEBUG [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:53,595 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.HStore(327): Store=94761791c1172512c89bb88a26d6fda0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:16:53,595 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:53,596 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:16:53,596 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94761791c1172512c89bb88a26d6fda0 columnFamilyName B 2024-11-15T22:16:53,596 DEBUG [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:53,597 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.HStore(327): Store=94761791c1172512c89bb88a26d6fda0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:16:53,597 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:53,612 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:16:53,613 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94761791c1172512c89bb88a26d6fda0 columnFamilyName C 2024-11-15T22:16:53,613 DEBUG [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:53,613 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.HStore(327): Store=94761791c1172512c89bb88a26d6fda0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:16:53,614 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:53,614 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:53,615 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:53,616 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:53,628 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:53,628 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:53,631 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:16:53,636 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:53,645 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:16:53,645 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 94761791c1172512c89bb88a26d6fda0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69964097, jitterRate=0.042546287178993225}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:16:53,646 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:53,646 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 94761791c1172512c89bb88a26d6fda0: Running coprocessor pre-open hook at 1731709013578Writing region info on filesystem at 1731709013578Initializing all the Stores at 1731709013585 (+7 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731709013585Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731709013589 (+4 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731709013589Cleaning up temporary data from old regions at 1731709013628 (+39 ms)Running coprocessor post-open hooks at 1731709013646 (+18 ms)Region opened successfully at 1731709013646 2024-11-15T22:16:53,650 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., pid=81, masterSystemTime=1731709013573 2024-11-15T22:16:53,653 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:53,653 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:53,653 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=94761791c1172512c89bb88a26d6fda0, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:16:53,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:16:53,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-15T22:16:53,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure 94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 in 235 msec 2024-11-15T22:16:53,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-15T22:16:53,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94761791c1172512c89bb88a26d6fda0, ASSIGN in 402 msec 2024-11-15T22:16:53,662 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T22:16:53,668 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731709013662"}]},"ts":"1731709013662"} 2024-11-15T22:16:53,670 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T22:16:53,671 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T22:16:53,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 473 msec 2024-11-15T22:16:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-15T22:16:53,832 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:53,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22bc821c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:53,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:53,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:53,836 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:53,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16e73fb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:53,842 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d041db1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:53,846 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:53,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:53,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:53,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:53,851 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T22:16:53,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e611192d6313,34969,1731708925722 2024-11-15T22:16:53,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@32841ec2 2024-11-15T22:16:53,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T22:16:53,858 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T22:16:53,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:16:53,861 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:16:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T22:16:53,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742149_1325 (size=1097) 2024-11-15T22:16:53,920 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-15T22:16:53,920 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-15T22:16:53,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:16:53,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94761791c1172512c89bb88a26d6fda0, REOPEN/MOVE}] 2024-11-15T22:16:53,929 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94761791c1172512c89bb88a26d6fda0, REOPEN/MOVE 2024-11-15T22:16:53,930 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=94761791c1172512c89bb88a26d6fda0, regionState=CLOSING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:16:53,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94761791c1172512c89bb88a26d6fda0, REOPEN/MOVE because future has completed 2024-11-15T22:16:53,932 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:16:53,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:16:54,085 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,085 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:16:54,085 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 94761791c1172512c89bb88a26d6fda0, disabling compactions & flushes 2024-11-15T22:16:54,085 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:54,085 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:54,085 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. after waiting 0 ms 2024-11-15T22:16:54,085 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:54,089 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:16:54,089 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:54,089 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 94761791c1172512c89bb88a26d6fda0: Waiting for close lock at 1731709014085Running coprocessor pre-close hooks at 1731709014085Disabling compacts and flushes for region at 1731709014085Disabling writes for close at 1731709014085Writing region close event to WAL at 1731709014086 (+1 ms)Running coprocessor post-close hooks at 1731709014089 (+3 ms)Closed at 1731709014089 2024-11-15T22:16:54,089 WARN [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: 94761791c1172512c89bb88a26d6fda0 to self. 2024-11-15T22:16:54,091 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,091 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=94761791c1172512c89bb88a26d6fda0, regionState=CLOSED 2024-11-15T22:16:54,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:16:54,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-15T22:16:54,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 in 167 msec 2024-11-15T22:16:54,102 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94761791c1172512c89bb88a26d6fda0, REOPEN/MOVE; state=CLOSED, location=e611192d6313,38779,1731708927034; forceNewPlan=false, retain=true 2024-11-15T22:16:54,254 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=94761791c1172512c89bb88a26d6fda0, regionState=OPENING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:16:54,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94761791c1172512c89bb88a26d6fda0, REOPEN/MOVE because future has completed 2024-11-15T22:16:54,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:16:54,418 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:54,418 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7752): Opening region: {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:16:54,418 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,418 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:16:54,419 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7794): checking encryption for 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,419 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7797): checking classloading for 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,420 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,421 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:16:54,421 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94761791c1172512c89bb88a26d6fda0 columnFamilyName A 2024-11-15T22:16:54,424 DEBUG [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:54,424 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.HStore(327): Store=94761791c1172512c89bb88a26d6fda0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:16:54,425 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,426 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:16:54,426 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94761791c1172512c89bb88a26d6fda0 columnFamilyName B 2024-11-15T22:16:54,426 DEBUG [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:54,427 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.HStore(327): Store=94761791c1172512c89bb88a26d6fda0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:16:54,427 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,427 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:16:54,427 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94761791c1172512c89bb88a26d6fda0 columnFamilyName C 2024-11-15T22:16:54,427 DEBUG [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:54,428 INFO [StoreOpener-94761791c1172512c89bb88a26d6fda0-1 {}] regionserver.HStore(327): Store=94761791c1172512c89bb88a26d6fda0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:16:54,428 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:54,428 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1038): replaying wal for 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,429 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,430 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,431 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1048): stopping wal replay for 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,431 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1060): Cleaning up temporary data for 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,431 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:16:54,433 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1093): writing seq id for 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,433 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1114): Opened 94761791c1172512c89bb88a26d6fda0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73963156, jitterRate=0.10213690996170044}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:16:54,433 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,434 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1006): Region open journal for 94761791c1172512c89bb88a26d6fda0: Running coprocessor pre-open hook at 1731709014419Writing region info on filesystem at 1731709014419Initializing all the Stores at 1731709014419Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731709014419Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731709014420 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731709014420Cleaning up temporary data from old regions at 1731709014431 (+11 ms)Running coprocessor post-open hooks at 1731709014433 (+2 ms)Region opened successfully at 1731709014434 (+1 ms) 2024-11-15T22:16:54,435 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., pid=86, masterSystemTime=1731709014415 2024-11-15T22:16:54,436 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:54,436 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:54,437 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=94761791c1172512c89bb88a26d6fda0, regionState=OPEN, openSeqNum=5, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:16:54,438 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:16:54,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=84 2024-11-15T22:16:54,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 in 178 msec 2024-11-15T22:16:54,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-15T22:16:54,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94761791c1172512c89bb88a26d6fda0, REOPEN/MOVE in 512 msec 2024-11-15T22:16:54,448 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-15T22:16:54,448 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 522 msec 2024-11-15T22:16:54,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 587 msec 2024-11-15T22:16:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T22:16:54,453 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:54,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf842e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:54,454 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:54,455 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56c094db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:54,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:54,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:54,457 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:54,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21370f73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f539a03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,465 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:54,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:54,466 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a2d6ef7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:54,468 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:54,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37e0811a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ef673d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,472 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:54,472 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:54,473 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f2172e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:54,475 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103a6343, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc7c0e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,489 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:54,490 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:54,493 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:54,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:54,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:54,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d611a99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:54,495 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89819c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c5958f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,520 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:54,521 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:54,522 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:54,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:54,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:54,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1714e2c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:54,525 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:54,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc8e999, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d0f7be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:54,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:54,535 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:54,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:54,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:54,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5011c746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:54,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:54,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:54,537 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:54,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63436fb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a06cc9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:54,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:54,544 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@170f2068, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:54,546 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:54,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20895db9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5990cd3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:54,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:54,549 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:54,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37bad59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:54,552 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:54,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f6d0b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8367c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:54,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:54,555 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67b3fad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:54,558 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:54,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38a7f7cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df1ab09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:16:54,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:16:54,565 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:16:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:16:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:16:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6117c140, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:16:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:16:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:16:54,569 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:16:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63ee83cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:16:54,584 DEBUG [Thread-1502 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:54,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:54,589 DEBUG [Thread-1504 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:54,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:54,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:54,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:54,595 DEBUG [Thread-1495 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:54,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:54,605 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-15T22:16:54,606 DEBUG [Thread-1491 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:54,608 DEBUG [Thread-1506 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T22:16:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:54,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:54,610 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:54,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:54,612 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:54,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:54,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:54,624 DEBUG [Thread-1489 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:54,625 DEBUG [Thread-1498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:54,626 DEBUG [Thread-1498 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:54,626 DEBUG [Thread-1498 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:54,627 DEBUG [Thread-1489 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:54,627 DEBUG [Thread-1489 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:54,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:54,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:54,637 DEBUG [Thread-1487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:54,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:54,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:54,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:54,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:54,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:54,644 DEBUG [Thread-1493 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:54,648 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:54,650 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T22:16:54,650 DEBUG [Thread-1500 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:16:54,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:16:54,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:16:54,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:54,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:16:54,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:16:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:54,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T22:16:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:16:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:16:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:16:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:54,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111541343d2deacf4aee9f7af8ee9bee59ef_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709014647/Put/seqid=0 2024-11-15T22:16:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T22:16:54,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742150_1326 (size=12154) 2024-11-15T22:16:54,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731709014709, max timestamp=1731709014727Num uniques 859; 2024-11-15T22:16:54,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T22:16:54,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731709014709, max timestamp=1731709014728Num uniques 859; 2024-11-15T22:16:54,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731709014709, max timestamp=1731709014728Num uniques 859; 2024-11-15T22:16:54,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T22:16:54,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T22:16:54,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-15T22:16:54,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:54,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:16:54,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:54,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:54,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:54,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709074767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:54,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709074769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:54,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709074772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:54,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58448 deadline: 1731709074778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:54,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:54,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709074778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:54,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709074881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:54,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709074882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:54,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709074892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:54,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58448 deadline: 1731709074901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:54,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709074901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:54,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-15T22:16:54,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:54,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:16:54,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:54,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:54,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T22:16:55,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-15T22:16:55,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:55,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:16:55,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:55,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709075101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:55,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:55,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709075105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:55,125 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:55,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:55,129 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111541343d2deacf4aee9f7af8ee9bee59ef_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111541343d2deacf4aee9f7af8ee9bee59ef_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709075112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:55,130 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/3ab77223d5bf426a860effbe14ba018f, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:16:55,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/3ab77223d5bf426a860effbe14ba018f is 175, key is test_row_0/A:col10/1731709014647/Put/seqid=0 2024-11-15T22:16:55,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58448 deadline: 1731709075122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:55,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709075122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:55,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742151_1327 (size=30955) 2024-11-15T22:16:55,160 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/3ab77223d5bf426a860effbe14ba018f 2024-11-15T22:16:55,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/bc8325c2f8eb4da0a770c8658caba681 is 50, key is test_row_0/B:col10/1731709014647/Put/seqid=0 2024-11-15T22:16:55,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742152_1328 (size=12001) 2024-11-15T22:16:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T22:16:55,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-15T22:16:55,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:55,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:16:55,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:55,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-15T22:16:55,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:55,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:16:55,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:55,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709075421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:55,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:55,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709075431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:55,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:55,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709075431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:55,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:55,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709075442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:55,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:55,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58448 deadline: 1731709075444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:55,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:55,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:55,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-15T22:16:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:16:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:55,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/bc8325c2f8eb4da0a770c8658caba681 2024-11-15T22:16:55,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/da486add07b14a0e984a45f32e00833d is 50, key is test_row_0/C:col10/1731709014647/Put/seqid=0 2024-11-15T22:16:55,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742153_1329 (size=12001) 2024-11-15T22:16:55,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-15T22:16:55,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:55,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:16:55,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:55,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T22:16:55,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-15T22:16:55,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:55,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:16:55,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:55,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:55,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709075941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:55,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709075951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:55,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709075952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:55,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709075961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:55,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58448 deadline: 1731709075961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:56,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-15T22:16:56,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:56,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:16:56,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:56,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:56,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:56,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/da486add07b14a0e984a45f32e00833d 2024-11-15T22:16:56,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/3ab77223d5bf426a860effbe14ba018f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/3ab77223d5bf426a860effbe14ba018f 2024-11-15T22:16:56,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/3ab77223d5bf426a860effbe14ba018f, entries=150, sequenceid=15, filesize=30.2 K 2024-11-15T22:16:56,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/bc8325c2f8eb4da0a770c8658caba681 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/bc8325c2f8eb4da0a770c8658caba681 2024-11-15T22:16:56,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/bc8325c2f8eb4da0a770c8658caba681, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T22:16:56,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/da486add07b14a0e984a45f32e00833d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/da486add07b14a0e984a45f32e00833d 2024-11-15T22:16:56,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/da486add07b14a0e984a45f32e00833d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T22:16:56,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for 94761791c1172512c89bb88a26d6fda0 in 1417ms, sequenceid=15, compaction requested=false 2024-11-15T22:16:56,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:16:56,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-15T22:16:56,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:56,159 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-15T22:16:56,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:16:56,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:56,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:16:56,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:56,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:16:56,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:56,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115eb4060d8ce6a4d52b23f7c1275da3829_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709014768/Put/seqid=0 2024-11-15T22:16:56,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742154_1330 (size=12154) 2024-11-15T22:16:56,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:56,588 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115eb4060d8ce6a4d52b23f7c1275da3829_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115eb4060d8ce6a4d52b23f7c1275da3829_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:56,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/53c631a4ac154d28b33ac92e397b4fbb, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:16:56,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/53c631a4ac154d28b33ac92e397b4fbb is 175, key is test_row_0/A:col10/1731709014768/Put/seqid=0 2024-11-15T22:16:56,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742155_1331 (size=30955) 2024-11-15T22:16:56,594 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/53c631a4ac154d28b33ac92e397b4fbb 2024-11-15T22:16:56,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/7516347337984f3dbd703fdcd0c0cb57 is 50, key is test_row_0/B:col10/1731709014768/Put/seqid=0 2024-11-15T22:16:56,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742156_1332 (size=12001) 2024-11-15T22:16:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T22:16:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:56,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:16:56,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709076974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:56,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:56,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58448 deadline: 1731709076982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:56,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709076984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:56,991 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in 94761791c1172512c89bb88a26d6fda0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:16:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:56,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:56,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709076986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:56,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:56,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:56,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709076986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:56,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:56,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:56,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:56,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,007 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/7516347337984f3dbd703fdcd0c0cb57 2024-11-15T22:16:57,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/69cc6fab5d7844c8a8a3f1c9c36da857 is 50, key is test_row_0/C:col10/1731709014768/Put/seqid=0 2024-11-15T22:16:57,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742157_1333 (size=12001) 2024-11-15T22:16:57,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709077095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709077101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709077102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709077102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709077312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709077312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709077321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709077321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,427 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/69cc6fab5d7844c8a8a3f1c9c36da857 2024-11-15T22:16:57,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/53c631a4ac154d28b33ac92e397b4fbb as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/53c631a4ac154d28b33ac92e397b4fbb 2024-11-15T22:16:57,436 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/53c631a4ac154d28b33ac92e397b4fbb, entries=150, sequenceid=50, filesize=30.2 K 2024-11-15T22:16:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/7516347337984f3dbd703fdcd0c0cb57 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/7516347337984f3dbd703fdcd0c0cb57 2024-11-15T22:16:57,443 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/7516347337984f3dbd703fdcd0c0cb57, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T22:16:57,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/69cc6fab5d7844c8a8a3f1c9c36da857 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/69cc6fab5d7844c8a8a3f1c9c36da857 2024-11-15T22:16:57,450 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/69cc6fab5d7844c8a8a3f1c9c36da857, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T22:16:57,451 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=53.67 KB/54960 for 94761791c1172512c89bb88a26d6fda0 in 1292ms, sequenceid=50, compaction requested=false 2024-11-15T22:16:57,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:16:57,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:57,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-15T22:16:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-15T22:16:57,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-15T22:16:57,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8410 sec 2024-11-15T22:16:57,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 2.8510 sec 2024-11-15T22:16:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:57,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T22:16:57,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:16:57,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:57,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:16:57,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:57,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:16:57,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:57,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115aa583939d163495b96043c658a20e727_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709016959/Put/seqid=0 2024-11-15T22:16:57,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742158_1334 (size=14594) 2024-11-15T22:16:57,697 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:57,701 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115aa583939d163495b96043c658a20e727_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115aa583939d163495b96043c658a20e727_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:57,702 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/a4afe824a7bd41cb8be2f98b4aaccdef, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:16:57,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/a4afe824a7bd41cb8be2f98b4aaccdef is 175, key is test_row_0/A:col10/1731709016959/Put/seqid=0 2024-11-15T22:16:57,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709077686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709077687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709077688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709077695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742159_1335 (size=39549) 2024-11-15T22:16:57,740 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/a4afe824a7bd41cb8be2f98b4aaccdef 2024-11-15T22:16:57,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/d0238f70861541e2a3661ab3fe683524 is 50, key is test_row_0/B:col10/1731709016959/Put/seqid=0 2024-11-15T22:16:57,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742160_1336 (size=12001) 2024-11-15T22:16:57,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709077813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709077814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709077814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:57,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709077821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709078031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709078031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709078031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709078031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,067 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T22:16:58,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/d0238f70861541e2a3661ab3fe683524 2024-11-15T22:16:58,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/d69b7fe5c6a34477b5ccc9c0fe24f9cc is 50, key is test_row_0/C:col10/1731709016959/Put/seqid=0 2024-11-15T22:16:58,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742161_1337 (size=12001) 2024-11-15T22:16:58,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/d69b7fe5c6a34477b5ccc9c0fe24f9cc 2024-11-15T22:16:58,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/a4afe824a7bd41cb8be2f98b4aaccdef as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/a4afe824a7bd41cb8be2f98b4aaccdef 2024-11-15T22:16:58,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/a4afe824a7bd41cb8be2f98b4aaccdef, entries=200, sequenceid=62, filesize=38.6 K 2024-11-15T22:16:58,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/d0238f70861541e2a3661ab3fe683524 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/d0238f70861541e2a3661ab3fe683524 2024-11-15T22:16:58,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/d0238f70861541e2a3661ab3fe683524, entries=150, sequenceid=62, filesize=11.7 K 2024-11-15T22:16:58,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/d69b7fe5c6a34477b5ccc9c0fe24f9cc as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/d69b7fe5c6a34477b5ccc9c0fe24f9cc 2024-11-15T22:16:58,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/d69b7fe5c6a34477b5ccc9c0fe24f9cc, entries=150, sequenceid=62, filesize=11.7 K 2024-11-15T22:16:58,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 94761791c1172512c89bb88a26d6fda0 in 599ms, sequenceid=62, compaction requested=true 2024-11-15T22:16:58,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:16:58,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:16:58,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:58,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:16:58,227 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:58,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:58,227 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:58,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:16:58,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:58,228 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:58,228 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:58,228 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/A is initiating minor compaction (all files) 2024-11-15T22:16:58,228 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/B is initiating minor compaction (all files) 2024-11-15T22:16:58,228 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/B in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:58,228 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/A in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:58,228 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/bc8325c2f8eb4da0a770c8658caba681, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/7516347337984f3dbd703fdcd0c0cb57, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/d0238f70861541e2a3661ab3fe683524] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=35.2 K 2024-11-15T22:16:58,228 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/3ab77223d5bf426a860effbe14ba018f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/53c631a4ac154d28b33ac92e397b4fbb, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/a4afe824a7bd41cb8be2f98b4aaccdef] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=99.1 K 2024-11-15T22:16:58,228 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:58,228 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/3ab77223d5bf426a860effbe14ba018f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/53c631a4ac154d28b33ac92e397b4fbb, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/a4afe824a7bd41cb8be2f98b4aaccdef] 2024-11-15T22:16:58,228 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting bc8325c2f8eb4da0a770c8658caba681, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731709014609 2024-11-15T22:16:58,228 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ab77223d5bf426a860effbe14ba018f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731709014609 2024-11-15T22:16:58,228 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 7516347337984f3dbd703fdcd0c0cb57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731709014746 2024-11-15T22:16:58,228 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53c631a4ac154d28b33ac92e397b4fbb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731709014746 2024-11-15T22:16:58,229 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d0238f70861541e2a3661ab3fe683524, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731709016959 2024-11-15T22:16:58,229 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4afe824a7bd41cb8be2f98b4aaccdef, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731709016959 2024-11-15T22:16:58,245 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#B#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:58,246 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/8d37a8d72f504f4db75409dd639e86a6 is 50, key is test_row_0/B:col10/1731709016959/Put/seqid=0 2024-11-15T22:16:58,247 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:16:58,249 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411155afaab01fe0442d2ac14b129403d77e7_94761791c1172512c89bb88a26d6fda0 store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:16:58,251 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411155afaab01fe0442d2ac14b129403d77e7_94761791c1172512c89bb88a26d6fda0, store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:16:58,252 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155afaab01fe0442d2ac14b129403d77e7_94761791c1172512c89bb88a26d6fda0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:16:58,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742162_1338 (size=12104) 2024-11-15T22:16:58,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742163_1339 (size=4469) 2024-11-15T22:16:58,259 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#A#compaction#289 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:58,259 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/6369fbc72af74c1896dd9cc0264e84d0 is 175, key is test_row_0/A:col10/1731709016959/Put/seqid=0 2024-11-15T22:16:58,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742164_1340 (size=31058) 2024-11-15T22:16:58,282 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/6369fbc72af74c1896dd9cc0264e84d0 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/6369fbc72af74c1896dd9cc0264e84d0 2024-11-15T22:16:58,290 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/A of 94761791c1172512c89bb88a26d6fda0 into 6369fbc72af74c1896dd9cc0264e84d0(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:58,290 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:16:58,290 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/A, priority=13, startTime=1731709018227; duration=0sec 2024-11-15T22:16:58,290 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:16:58,290 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:A 2024-11-15T22:16:58,291 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:16:58,291 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:16:58,292 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/C is initiating minor compaction (all files) 2024-11-15T22:16:58,292 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/C in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:58,292 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/da486add07b14a0e984a45f32e00833d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/69cc6fab5d7844c8a8a3f1c9c36da857, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/d69b7fe5c6a34477b5ccc9c0fe24f9cc] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=35.2 K 2024-11-15T22:16:58,292 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting da486add07b14a0e984a45f32e00833d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731709014609 2024-11-15T22:16:58,292 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69cc6fab5d7844c8a8a3f1c9c36da857, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731709014746 2024-11-15T22:16:58,293 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting d69b7fe5c6a34477b5ccc9c0fe24f9cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731709016959 2024-11-15T22:16:58,305 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#C#compaction#290 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:16:58,308 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/204ef1bb6ac24764984fbf21bcd84f6a is 50, key is test_row_0/C:col10/1731709016959/Put/seqid=0 2024-11-15T22:16:58,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742165_1341 (size=12104) 2024-11-15T22:16:58,323 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/204ef1bb6ac24764984fbf21bcd84f6a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/204ef1bb6ac24764984fbf21bcd84f6a 2024-11-15T22:16:58,340 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/C of 94761791c1172512c89bb88a26d6fda0 into 204ef1bb6ac24764984fbf21bcd84f6a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:58,340 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:16:58,340 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/C, priority=13, startTime=1731709018227; duration=0sec 2024-11-15T22:16:58,340 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:58,340 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:C 2024-11-15T22:16:58,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T22:16:58,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:16:58,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:58,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:16:58,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:58,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:16:58,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:16:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:58,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111584fb2000294741409004278e31e7b242_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709017673/Put/seqid=0 2024-11-15T22:16:58,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709078374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709078375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709078376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709078377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742166_1342 (size=14594) 2024-11-15T22:16:58,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709078493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709078493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709078495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709078496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,666 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/8d37a8d72f504f4db75409dd639e86a6 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/8d37a8d72f504f4db75409dd639e86a6 2024-11-15T22:16:58,681 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/B of 94761791c1172512c89bb88a26d6fda0 into 8d37a8d72f504f4db75409dd639e86a6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:16:58,681 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:16:58,681 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/B, priority=13, startTime=1731709018227; duration=0sec 2024-11-15T22:16:58,681 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:16:58,681 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:B 2024-11-15T22:16:58,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709078711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709078712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709078713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709078721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:58,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T22:16:58,782 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:16:58,782 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:16:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-15T22:16:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-15T22:16:58,784 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:16:58,784 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:16:58,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:16:58,789 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:16:58,792 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111584fb2000294741409004278e31e7b242_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111584fb2000294741409004278e31e7b242_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:16:58,792 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/7006aa1a138e498bb283ee62b9216137, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:16:58,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/7006aa1a138e498bb283ee62b9216137 is 175, key is test_row_0/A:col10/1731709017673/Put/seqid=0 2024-11-15T22:16:58,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742167_1343 (size=39549) 2024-11-15T22:16:58,802 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/7006aa1a138e498bb283ee62b9216137 2024-11-15T22:16:58,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/1b8e780adfd249a4b386866ca7d48102 is 50, key is test_row_0/B:col10/1731709017673/Put/seqid=0 2024-11-15T22:16:58,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742168_1344 (size=12001) 2024-11-15T22:16:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-15T22:16:58,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T22:16:58,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:58,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:16:58,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:58,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:58,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:58,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:59,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58448 deadline: 1731709079011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:59,020 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in 94761791c1172512c89bb88a26d6fda0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:16:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:59,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709079032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:59,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709079032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:59,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709079032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:59,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709079037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:59,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T22:16:59,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:59,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:16:59,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:59,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:59,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:59,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-15T22:16:59,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/1b8e780adfd249a4b386866ca7d48102 2024-11-15T22:16:59,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/bdacf545453c4c0e86cc7a3ced1ab3d9 is 50, key is test_row_0/C:col10/1731709017673/Put/seqid=0 2024-11-15T22:16:59,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T22:16:59,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:59,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:16:59,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:59,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:59,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:59,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T22:16:59,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:59,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:16:59,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:59,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:59,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-15T22:16:59,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T22:16:59,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:59,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:16:59,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:59,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:59,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:16:59,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709079551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:59,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709079552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:59,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709079552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:16:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709079552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:16:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:16:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:16:59,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742169_1345 (size=12001) 2024-11-15T22:16:59,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/bdacf545453c4c0e86cc7a3ced1ab3d9 2024-11-15T22:16:59,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/7006aa1a138e498bb283ee62b9216137 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/7006aa1a138e498bb283ee62b9216137 2024-11-15T22:16:59,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/7006aa1a138e498bb283ee62b9216137, entries=200, sequenceid=85, filesize=38.6 K 2024-11-15T22:16:59,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/1b8e780adfd249a4b386866ca7d48102 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/1b8e780adfd249a4b386866ca7d48102 2024-11-15T22:16:59,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/1b8e780adfd249a4b386866ca7d48102, entries=150, sequenceid=85, filesize=11.7 K 2024-11-15T22:16:59,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/bdacf545453c4c0e86cc7a3ced1ab3d9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/bdacf545453c4c0e86cc7a3ced1ab3d9 2024-11-15T22:16:59,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/bdacf545453c4c0e86cc7a3ced1ab3d9, entries=150, sequenceid=85, filesize=11.7 K 2024-11-15T22:16:59,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 94761791c1172512c89bb88a26d6fda0 in 1247ms, sequenceid=85, compaction requested=false 2024-11-15T22:16:59,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:16:59,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T22:16:59,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:16:59,706 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T22:16:59,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:16:59,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:59,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:16:59,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:59,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:16:59,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:16:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b4350214ec1e4072b74b860c34a07521_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709018375/Put/seqid=0 2024-11-15T22:16:59,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742170_1346 (size=12154) 2024-11-15T22:16:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-15T22:17:00,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:00,130 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b4350214ec1e4072b74b860c34a07521_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b4350214ec1e4072b74b860c34a07521_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:00,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/e843f4a864c245ceb36d330bec940524, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:00,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/e843f4a864c245ceb36d330bec940524 is 175, key is test_row_0/A:col10/1731709018375/Put/seqid=0 2024-11-15T22:17:00,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742171_1347 (size=30955) 2024-11-15T22:17:00,537 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/e843f4a864c245ceb36d330bec940524 2024-11-15T22:17:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/90b45ce42dab4586ab92fe93de4bddea is 50, key is test_row_0/B:col10/1731709018375/Put/seqid=0 2024-11-15T22:17:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:00,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:00,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742172_1348 (size=12001) 2024-11-15T22:17:00,584 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/90b45ce42dab4586ab92fe93de4bddea 2024-11-15T22:17:00,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/46a7a177224b461eb4a107d090dce3c3 is 50, key is test_row_0/C:col10/1731709018375/Put/seqid=0 2024-11-15T22:17:00,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742173_1349 (size=12001) 2024-11-15T22:17:00,627 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/46a7a177224b461eb4a107d090dce3c3 2024-11-15T22:17:00,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/e843f4a864c245ceb36d330bec940524 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/e843f4a864c245ceb36d330bec940524 2024-11-15T22:17:00,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709080635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:00,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21014, totalHeapSize=47976, min timestamp=1731709020600, max timestamp=1731709020653Num uniques 859; 2024-11-15T22:17:00,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=21014, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-15T22:17:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:00,669 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/e843f4a864c245ceb36d330bec940524, entries=150, sequenceid=97, filesize=30.2 K 2024-11-15T22:17:00,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/90b45ce42dab4586ab92fe93de4bddea as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/90b45ce42dab4586ab92fe93de4bddea 2024-11-15T22:17:00,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709080661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:00,680 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/90b45ce42dab4586ab92fe93de4bddea, entries=150, sequenceid=97, filesize=11.7 K 2024-11-15T22:17:00,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/46a7a177224b461eb4a107d090dce3c3 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/46a7a177224b461eb4a107d090dce3c3 2024-11-15T22:17:00,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709080679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709080679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:00,694 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/46a7a177224b461eb4a107d090dce3c3, entries=150, sequenceid=97, filesize=11.7 K 2024-11-15T22:17:00,695 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=123.00 KB/125950 for 94761791c1172512c89bb88a26d6fda0 in 989ms, sequenceid=97, compaction requested=true 2024-11-15T22:17:00,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:00,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:00,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-15T22:17:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-15T22:17:00,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-15T22:17:00,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9130 sec 2024-11-15T22:17:00,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 1.9180 sec 2024-11-15T22:17:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:00,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=129.71 KB heapSize=372.27 KB 2024-11-15T22:17:00,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:17:00,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:00,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:17:00,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:00,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:17:00,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:00,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158165fe5a6c8e435099e15cc379a74bd2_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709020767/Put/seqid=0 2024-11-15T22:17:00,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742174_1350 (size=14594) 2024-11-15T22:17:00,828 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:00,836 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158165fe5a6c8e435099e15cc379a74bd2_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158165fe5a6c8e435099e15cc379a74bd2_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:00,838 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/148fdad423924675b7f7b575b8ee3da9, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:00,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/148fdad423924675b7f7b575b8ee3da9 is 175, key is test_row_0/A:col10/1731709020767/Put/seqid=0 2024-11-15T22:17:00,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742175_1351 (size=39549) 2024-11-15T22:17:00,856 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/148fdad423924675b7f7b575b8ee3da9 2024-11-15T22:17:00,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/489db5cd39e642b5917789df0950e4c6 is 50, key is test_row_0/B:col10/1731709020767/Put/seqid=0 2024-11-15T22:17:00,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742176_1352 (size=12001) 2024-11-15T22:17:00,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709080834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:00,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709080893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:00,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709080893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709080893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-15T22:17:00,931 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:00,932 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:17:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-15T22:17:00,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T22:17:00,935 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:17:00,936 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:17:00,936 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:17:01,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709081012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709081012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709081021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709081021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T22:17:01,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-15T22:17:01,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:01,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:01,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:01,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:01,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:01,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709081231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709081231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-15T22:17:01,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:01,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:01,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:01,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:01,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:01,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709081242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709081242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T22:17:01,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/489db5cd39e642b5917789df0950e4c6 2024-11-15T22:17:01,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/60e30d3cd7ff4aa1968b39d76e416e2d is 50, key is test_row_0/C:col10/1731709020767/Put/seqid=0 2024-11-15T22:17:01,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742177_1353 (size=12001) 2024-11-15T22:17:01,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/60e30d3cd7ff4aa1968b39d76e416e2d 2024-11-15T22:17:01,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/148fdad423924675b7f7b575b8ee3da9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/148fdad423924675b7f7b575b8ee3da9 2024-11-15T22:17:01,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/148fdad423924675b7f7b575b8ee3da9, entries=200, sequenceid=122, filesize=38.6 K 2024-11-15T22:17:01,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/489db5cd39e642b5917789df0950e4c6 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/489db5cd39e642b5917789df0950e4c6 2024-11-15T22:17:01,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/489db5cd39e642b5917789df0950e4c6, entries=150, sequenceid=122, filesize=11.7 K 2024-11-15T22:17:01,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/60e30d3cd7ff4aa1968b39d76e416e2d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/60e30d3cd7ff4aa1968b39d76e416e2d 2024-11-15T22:17:01,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/60e30d3cd7ff4aa1968b39d76e416e2d, entries=150, sequenceid=122, filesize=11.7 K 2024-11-15T22:17:01,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~372.22 KB/381152, currentSize=60.38 KB/61830 for 94761791c1172512c89bb88a26d6fda0 in 581ms, sequenceid=122, compaction requested=true 2024-11-15T22:17:01,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:01,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:01,350 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:17:01,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:01,351 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:17:01,353 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:17:01,353 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/A is initiating minor compaction (all files) 2024-11-15T22:17:01,353 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/A in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:01,353 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/6369fbc72af74c1896dd9cc0264e84d0, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/7006aa1a138e498bb283ee62b9216137, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/e843f4a864c245ceb36d330bec940524, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/148fdad423924675b7f7b575b8ee3da9] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=137.8 K 2024-11-15T22:17:01,353 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:01,353 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/6369fbc72af74c1896dd9cc0264e84d0, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/7006aa1a138e498bb283ee62b9216137, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/e843f4a864c245ceb36d330bec940524, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/148fdad423924675b7f7b575b8ee3da9] 2024-11-15T22:17:01,354 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6369fbc72af74c1896dd9cc0264e84d0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731709016959 2024-11-15T22:17:01,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:01,354 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7006aa1a138e498bb283ee62b9216137, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731709017673 2024-11-15T22:17:01,354 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:17:01,354 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/B is initiating minor compaction (all files) 2024-11-15T22:17:01,354 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/B in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:01,354 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/8d37a8d72f504f4db75409dd639e86a6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/1b8e780adfd249a4b386866ca7d48102, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/90b45ce42dab4586ab92fe93de4bddea, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/489db5cd39e642b5917789df0950e4c6] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=47.0 K 2024-11-15T22:17:01,354 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting e843f4a864c245ceb36d330bec940524, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731709018371 2024-11-15T22:17:01,355 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d37a8d72f504f4db75409dd639e86a6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731709016959 2024-11-15T22:17:01,355 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 148fdad423924675b7f7b575b8ee3da9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731709020617 2024-11-15T22:17:01,355 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b8e780adfd249a4b386866ca7d48102, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731709017673 2024-11-15T22:17:01,356 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 90b45ce42dab4586ab92fe93de4bddea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731709018371 2024-11-15T22:17:01,356 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 489db5cd39e642b5917789df0950e4c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731709020617 2024-11-15T22:17:01,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:01,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:01,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:01,368 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:01,368 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/12846108594b412bb124734ec671533a is 50, key is test_row_0/B:col10/1731709020767/Put/seqid=0 2024-11-15T22:17:01,384 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:01,388 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111510df04c0febf42e49ca50497749899df_94761791c1172512c89bb88a26d6fda0 store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:01,390 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111510df04c0febf42e49ca50497749899df_94761791c1172512c89bb88a26d6fda0, store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:01,390 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111510df04c0febf42e49ca50497749899df_94761791c1172512c89bb88a26d6fda0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:01,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-15T22:17:01,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:01,396 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T22:17:01,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:17:01,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:01,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:17:01,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:01,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:17:01,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:01,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742178_1354 (size=12241) 2024-11-15T22:17:01,412 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/12846108594b412bb124734ec671533a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/12846108594b412bb124734ec671533a 2024-11-15T22:17:01,417 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/B of 94761791c1172512c89bb88a26d6fda0 into 12846108594b412bb124734ec671533a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:01,417 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:01,417 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/B, priority=12, startTime=1731709021350; duration=0sec 2024-11-15T22:17:01,417 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:01,417 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:B 2024-11-15T22:17:01,417 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:17:01,419 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:17:01,419 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/C is initiating minor compaction (all files) 2024-11-15T22:17:01,419 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/C in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:01,419 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/204ef1bb6ac24764984fbf21bcd84f6a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/bdacf545453c4c0e86cc7a3ced1ab3d9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/46a7a177224b461eb4a107d090dce3c3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/60e30d3cd7ff4aa1968b39d76e416e2d] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=47.0 K 2024-11-15T22:17:01,419 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 204ef1bb6ac24764984fbf21bcd84f6a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731709016959 2024-11-15T22:17:01,420 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting bdacf545453c4c0e86cc7a3ced1ab3d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731709017673 2024-11-15T22:17:01,420 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 46a7a177224b461eb4a107d090dce3c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731709018371 2024-11-15T22:17:01,420 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 60e30d3cd7ff4aa1968b39d76e416e2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731709020617 2024-11-15T22:17:01,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742179_1355 (size=4469) 2024-11-15T22:17:01,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151a1da8d19594470bbd3dad392c5a237d_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709020850/Put/seqid=0 2024-11-15T22:17:01,438 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#C#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:01,439 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/b25174f22a064b9a8a1efe1cbe3edd8c is 50, key is test_row_0/C:col10/1731709020767/Put/seqid=0 2024-11-15T22:17:01,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742180_1356 (size=12304) 2024-11-15T22:17:01,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742181_1357 (size=12241) 2024-11-15T22:17:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:01,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T22:17:01,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709081580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709081585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709081585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709081592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709081702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709081702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709081702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709081712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,834 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#A#compaction#301 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:01,834 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/d51788963cda468bae4908fe8a4d9415 is 175, key is test_row_0/A:col10/1731709020767/Put/seqid=0 2024-11-15T22:17:01,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:01,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742182_1358 (size=31195) 2024-11-15T22:17:01,859 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151a1da8d19594470bbd3dad392c5a237d_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151a1da8d19594470bbd3dad392c5a237d_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:01,861 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/b25174f22a064b9a8a1efe1cbe3edd8c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/b25174f22a064b9a8a1efe1cbe3edd8c 2024-11-15T22:17:01,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/3fe637632a2945a8851fb933c759c888, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:01,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/3fe637632a2945a8851fb933c759c888 is 175, key is test_row_0/A:col10/1731709020850/Put/seqid=0 2024-11-15T22:17:01,864 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/d51788963cda468bae4908fe8a4d9415 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d51788963cda468bae4908fe8a4d9415 2024-11-15T22:17:01,881 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/A of 94761791c1172512c89bb88a26d6fda0 into d51788963cda468bae4908fe8a4d9415(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:01,881 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:01,881 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/A, priority=12, startTime=1731709021350; duration=0sec 2024-11-15T22:17:01,881 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/C of 94761791c1172512c89bb88a26d6fda0 into b25174f22a064b9a8a1efe1cbe3edd8c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:01,881 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:01,881 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:01,881 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/C, priority=12, startTime=1731709021357; duration=0sec 2024-11-15T22:17:01,881 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:A 2024-11-15T22:17:01,881 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:01,881 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:C 2024-11-15T22:17:01,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742183_1359 (size=31105) 2024-11-15T22:17:01,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709081922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709081922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709081923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:01,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709081931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T22:17:02,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709082241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:02,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709082241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:02,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:02,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709082241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:02,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:02,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709082252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:02,284 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/3fe637632a2945a8851fb933c759c888 2024-11-15T22:17:02,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/55de7b2d0e534ad3b8f78717908ea077 is 50, key is test_row_0/B:col10/1731709020850/Put/seqid=0 2024-11-15T22:17:02,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742184_1360 (size=12151) 2024-11-15T22:17:02,308 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/55de7b2d0e534ad3b8f78717908ea077 2024-11-15T22:17:02,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/9447968ce6084bab93c980a781d2f50f is 50, key is test_row_0/C:col10/1731709020850/Put/seqid=0 2024-11-15T22:17:02,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742185_1361 (size=12151) 2024-11-15T22:17:02,350 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/9447968ce6084bab93c980a781d2f50f 2024-11-15T22:17:02,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/3fe637632a2945a8851fb933c759c888 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/3fe637632a2945a8851fb933c759c888 2024-11-15T22:17:02,365 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/3fe637632a2945a8851fb933c759c888, entries=150, sequenceid=134, filesize=30.4 K 2024-11-15T22:17:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/55de7b2d0e534ad3b8f78717908ea077 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/55de7b2d0e534ad3b8f78717908ea077 2024-11-15T22:17:02,378 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/55de7b2d0e534ad3b8f78717908ea077, entries=150, sequenceid=134, filesize=11.9 K 2024-11-15T22:17:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/9447968ce6084bab93c980a781d2f50f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/9447968ce6084bab93c980a781d2f50f 2024-11-15T22:17:02,387 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/9447968ce6084bab93c980a781d2f50f, entries=150, sequenceid=134, filesize=11.9 K 2024-11-15T22:17:02,388 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 94761791c1172512c89bb88a26d6fda0 in 992ms, sequenceid=134, compaction requested=false 2024-11-15T22:17:02,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:02,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:02,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-15T22:17:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-15T22:17:02,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-15T22:17:02,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4550 sec 2024-11-15T22:17:02,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.4620 sec 2024-11-15T22:17:02,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731709021579, max timestamp=1731709022765Num uniques 859; 2024-11-15T22:17:02,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731709021579, max timestamp=1731709022765Num uniques 859; 2024-11-15T22:17:02,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:17:02,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:17:02,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731709021579, max timestamp=1731709022765Num uniques 859; 2024-11-15T22:17:02,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:17:02,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=60.38 KB heapSize=165.84 KB 2024-11-15T22:17:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:02,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:17:02,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:02,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:17:02,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:17:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:02,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159f2b427a0f7347dfa5014b4c8033f099_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709022777/Put/seqid=0 2024-11-15T22:17:02,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742186_1362 (size=12304) 2024-11-15T22:17:02,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709082839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:02,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709082840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:02,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709082841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:02,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709082852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:02,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709082961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:02,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709082961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:02,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709082961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:02,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709082961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:02,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:02,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:02,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:03,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:03,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58448 deadline: 1731709083051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:03,065 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in 94761791c1172512c89bb88a26d6fda0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8288 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:17:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T22:17:03,082 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:03,082 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:17:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-15T22:17:03,084 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:17:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T22:17:03,084 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:17:03,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:17:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T22:17:03,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709083181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709083181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709083181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:03,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709083181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:03,218 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:03,221 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159f2b427a0f7347dfa5014b4c8033f099_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159f2b427a0f7347dfa5014b4c8033f099_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:03,222 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/1cf9ae84262b4982955ab23cd039c8c8, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:03,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/1cf9ae84262b4982955ab23cd039c8c8 is 175, key is test_row_0/A:col10/1731709022777/Put/seqid=0 2024-11-15T22:17:03,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742187_1363 (size=31105) 2024-11-15T22:17:03,227 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=164, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/1cf9ae84262b4982955ab23cd039c8c8 2024-11-15T22:17:03,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/f40500ac4dca476381347703def7d06c is 50, key is test_row_0/B:col10/1731709022777/Put/seqid=0 2024-11-15T22:17:03,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T22:17:03,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:03,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:03,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:03,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:03,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:03,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742188_1364 (size=12151) 2024-11-15T22:17:03,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T22:17:03,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:03,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:03,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:03,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:03,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T22:17:03,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709083502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:03,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709083502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:03,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709083502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:03,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709083503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:03,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T22:17:03,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:03,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:03,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:03,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:03,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:03,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/f40500ac4dca476381347703def7d06c 2024-11-15T22:17:03,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/150a480a566b4f6f8381e97dae15dd19 is 50, key is test_row_0/C:col10/1731709022777/Put/seqid=0 2024-11-15T22:17:03,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T22:17:03,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:03,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:03,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:03,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:03,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T22:17:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:03,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742189_1365 (size=12151) 2024-11-15T22:17:03,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/150a480a566b4f6f8381e97dae15dd19 2024-11-15T22:17:03,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/1cf9ae84262b4982955ab23cd039c8c8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/1cf9ae84262b4982955ab23cd039c8c8 2024-11-15T22:17:03,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/1cf9ae84262b4982955ab23cd039c8c8, entries=150, sequenceid=164, filesize=30.4 K 2024-11-15T22:17:03,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/f40500ac4dca476381347703def7d06c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/f40500ac4dca476381347703def7d06c 2024-11-15T22:17:03,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/f40500ac4dca476381347703def7d06c, entries=150, sequenceid=164, filesize=11.9 K 2024-11-15T22:17:03,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/150a480a566b4f6f8381e97dae15dd19 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/150a480a566b4f6f8381e97dae15dd19 2024-11-15T22:17:03,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/150a480a566b4f6f8381e97dae15dd19, entries=150, sequenceid=164, filesize=11.9 K 2024-11-15T22:17:03,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~185.72 KB/190176, currentSize=114.05 KB/116790 for 94761791c1172512c89bb88a26d6fda0 in 993ms, sequenceid=164, compaction requested=true 2024-11-15T22:17:03,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:03,772 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:03,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:03,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:03,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:03,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:03,773 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:03,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:03,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:03,775 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:03,775 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:03,776 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/B is initiating minor compaction (all files) 2024-11-15T22:17:03,776 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/A is initiating minor compaction (all files) 2024-11-15T22:17:03,776 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/B in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:03,776 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/A in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:03,776 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/12846108594b412bb124734ec671533a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/55de7b2d0e534ad3b8f78717908ea077, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/f40500ac4dca476381347703def7d06c] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=35.7 K 2024-11-15T22:17:03,776 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d51788963cda468bae4908fe8a4d9415, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/3fe637632a2945a8851fb933c759c888, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/1cf9ae84262b4982955ab23cd039c8c8] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=91.2 K 2024-11-15T22:17:03,776 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:03,776 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d51788963cda468bae4908fe8a4d9415, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/3fe637632a2945a8851fb933c759c888, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/1cf9ae84262b4982955ab23cd039c8c8] 2024-11-15T22:17:03,776 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 12846108594b412bb124734ec671533a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731709020617 2024-11-15T22:17:03,776 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 55de7b2d0e534ad3b8f78717908ea077, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731709020822 2024-11-15T22:17:03,776 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting d51788963cda468bae4908fe8a4d9415, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731709020617 2024-11-15T22:17:03,777 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting f40500ac4dca476381347703def7d06c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731709022765 2024-11-15T22:17:03,777 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fe637632a2945a8851fb933c759c888, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731709020822 2024-11-15T22:17:03,778 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cf9ae84262b4982955ab23cd039c8c8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731709022765 2024-11-15T22:17:03,791 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#B#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:03,792 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/d07a0a36756247999bfe28dfe80561d7 is 50, key is test_row_0/B:col10/1731709022777/Put/seqid=0 2024-11-15T22:17:03,805 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:03,809 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115a6d9717c8bce47c7af4930191e1c6a4a_94761791c1172512c89bb88a26d6fda0 store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:03,811 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115a6d9717c8bce47c7af4930191e1c6a4a_94761791c1172512c89bb88a26d6fda0, store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:03,811 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a6d9717c8bce47c7af4930191e1c6a4a_94761791c1172512c89bb88a26d6fda0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:03,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742190_1366 (size=12493) 2024-11-15T22:17:03,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742191_1367 (size=4469) 2024-11-15T22:17:03,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T22:17:03,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:03,864 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T22:17:03,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:17:03,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:03,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:17:03,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:03,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:17:03,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:03,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154af17d9388a44c05bccd2d1f5fdb37b2_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709022850/Put/seqid=0 2024-11-15T22:17:03,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742192_1368 (size=12304) 2024-11-15T22:17:03,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:03,933 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154af17d9388a44c05bccd2d1f5fdb37b2_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154af17d9388a44c05bccd2d1f5fdb37b2_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/9d70cfdc6040447bb0536eb3f3deac0d, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:03,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/9d70cfdc6040447bb0536eb3f3deac0d is 175, key is test_row_0/A:col10/1731709022850/Put/seqid=0 2024-11-15T22:17:03,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742193_1369 (size=31105) 2024-11-15T22:17:04,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:04,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709084054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709084059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709084072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709084072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709084184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709084188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709084196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709084201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T22:17:04,248 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/d07a0a36756247999bfe28dfe80561d7 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/d07a0a36756247999bfe28dfe80561d7 2024-11-15T22:17:04,258 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/B of 94761791c1172512c89bb88a26d6fda0 into d07a0a36756247999bfe28dfe80561d7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:04,258 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:04,258 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/B, priority=13, startTime=1731709023772; duration=0sec 2024-11-15T22:17:04,258 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:04,259 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:B 2024-11-15T22:17:04,259 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#A#compaction#310 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:04,259 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:04,259 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/7e9ce2a5107a41aba83e970e9d7c4e52 is 175, key is test_row_0/A:col10/1731709022777/Put/seqid=0 2024-11-15T22:17:04,260 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:04,260 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/C is initiating minor compaction (all files) 2024-11-15T22:17:04,260 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/C in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:04,260 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/b25174f22a064b9a8a1efe1cbe3edd8c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/9447968ce6084bab93c980a781d2f50f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/150a480a566b4f6f8381e97dae15dd19] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=35.7 K 2024-11-15T22:17:04,261 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting b25174f22a064b9a8a1efe1cbe3edd8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731709020617 2024-11-15T22:17:04,261 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 9447968ce6084bab93c980a781d2f50f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731709020822 2024-11-15T22:17:04,261 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 150a480a566b4f6f8381e97dae15dd19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731709022765 2024-11-15T22:17:04,270 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#C#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:04,271 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/8f06802e78e94dbfbdbd31172f149a7c is 50, key is test_row_0/C:col10/1731709022777/Put/seqid=0 2024-11-15T22:17:04,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742194_1370 (size=31447) 2024-11-15T22:17:04,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742195_1371 (size=12493) 2024-11-15T22:17:04,306 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/8f06802e78e94dbfbdbd31172f149a7c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/8f06802e78e94dbfbdbd31172f149a7c 2024-11-15T22:17:04,314 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/C of 94761791c1172512c89bb88a26d6fda0 into 8f06802e78e94dbfbdbd31172f149a7c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:04,314 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:04,314 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/C, priority=13, startTime=1731709023773; duration=0sec 2024-11-15T22:17:04,314 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:04,314 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:C 2024-11-15T22:17:04,378 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/9d70cfdc6040447bb0536eb3f3deac0d 2024-11-15T22:17:04,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/1c8fc7c313fb4f98a6f09ac56d3352b6 is 50, key is test_row_0/B:col10/1731709022850/Put/seqid=0 2024-11-15T22:17:04,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709084401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709084413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709084422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709084422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742196_1372 (size=12151) 2024-11-15T22:17:04,683 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/7e9ce2a5107a41aba83e970e9d7c4e52 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/7e9ce2a5107a41aba83e970e9d7c4e52 2024-11-15T22:17:04,689 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/A of 94761791c1172512c89bb88a26d6fda0 into 7e9ce2a5107a41aba83e970e9d7c4e52(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:04,689 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:04,689 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/A, priority=13, startTime=1731709023772; duration=0sec 2024-11-15T22:17:04,689 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:04,689 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:A 2024-11-15T22:17:04,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709084723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709084732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709084732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709084742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:04,840 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/1c8fc7c313fb4f98a6f09ac56d3352b6 2024-11-15T22:17:04,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/8f00194263ca414682d9446cac5b3574 is 50, key is test_row_0/C:col10/1731709022850/Put/seqid=0 2024-11-15T22:17:04,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742197_1373 (size=12151) 2024-11-15T22:17:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T22:17:05,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709085242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:05,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709085242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:05,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:05,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:05,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:05,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709085242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:05,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709085251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:05,267 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/8f00194263ca414682d9446cac5b3574 2024-11-15T22:17:05,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/9d70cfdc6040447bb0536eb3f3deac0d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/9d70cfdc6040447bb0536eb3f3deac0d 2024-11-15T22:17:05,280 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/9d70cfdc6040447bb0536eb3f3deac0d, entries=150, sequenceid=184, filesize=30.4 K 2024-11-15T22:17:05,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/1c8fc7c313fb4f98a6f09ac56d3352b6 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/1c8fc7c313fb4f98a6f09ac56d3352b6 2024-11-15T22:17:05,288 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/1c8fc7c313fb4f98a6f09ac56d3352b6, entries=150, sequenceid=184, filesize=11.9 K 2024-11-15T22:17:05,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/8f00194263ca414682d9446cac5b3574 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/8f00194263ca414682d9446cac5b3574 2024-11-15T22:17:05,294 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/8f00194263ca414682d9446cac5b3574, entries=150, sequenceid=184, filesize=11.9 K 2024-11-15T22:17:05,295 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 94761791c1172512c89bb88a26d6fda0 in 1430ms, sequenceid=184, compaction requested=false 2024-11-15T22:17:05,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:05,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:05,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-15T22:17:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-15T22:17:05,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-15T22:17:05,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2120 sec 2024-11-15T22:17:05,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.2160 sec 2024-11-15T22:17:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:06,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T22:17:06,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:17:06,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:06,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:17:06,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:06,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:17:06,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:06,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d6fa0c245c5a4e0baaf08227bded4a27_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709024069/Put/seqid=0 2024-11-15T22:17:06,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709086349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:06,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709086353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:06,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:06,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742198_1374 (size=12304) 2024-11-15T22:17:06,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709086354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:06,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709086355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:06,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:06,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709086462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:06,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709086471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:06,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:06,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:06,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709086492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709086492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:06,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:06,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709086682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:06,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:06,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709086691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:06,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709086724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:06,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709086724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:06,778 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:06,798 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d6fa0c245c5a4e0baaf08227bded4a27_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d6fa0c245c5a4e0baaf08227bded4a27_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:06,803 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/d566dae6d00040bd9b01189a6a909393, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:06,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/d566dae6d00040bd9b01189a6a909393 is 175, key is test_row_0/A:col10/1731709024069/Put/seqid=0 2024-11-15T22:17:06,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742199_1375 (size=31105) 2024-11-15T22:17:06,872 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/d566dae6d00040bd9b01189a6a909393 2024-11-15T22:17:06,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/8d4969b0467e4814b2acc0107e526fa8 is 50, key is test_row_0/B:col10/1731709024069/Put/seqid=0 2024-11-15T22:17:06,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742200_1376 (size=12151) 2024-11-15T22:17:06,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/8d4969b0467e4814b2acc0107e526fa8 2024-11-15T22:17:06,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/103766b7fa96423eb53998f9c1f5ca1f is 50, key is test_row_0/C:col10/1731709024069/Put/seqid=0 2024-11-15T22:17:07,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742201_1377 (size=12151) 2024-11-15T22:17:07,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/103766b7fa96423eb53998f9c1f5ca1f 2024-11-15T22:17:07,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/d566dae6d00040bd9b01189a6a909393 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d566dae6d00040bd9b01189a6a909393 2024-11-15T22:17:07,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709087005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:07,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d566dae6d00040bd9b01189a6a909393, entries=150, sequenceid=201, filesize=30.4 K 2024-11-15T22:17:07,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/8d4969b0467e4814b2acc0107e526fa8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/8d4969b0467e4814b2acc0107e526fa8 2024-11-15T22:17:07,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709087012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:07,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709087048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:07,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709087048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:07,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/8d4969b0467e4814b2acc0107e526fa8, entries=150, sequenceid=201, filesize=11.9 K 2024-11-15T22:17:07,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/103766b7fa96423eb53998f9c1f5ca1f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/103766b7fa96423eb53998f9c1f5ca1f 2024-11-15T22:17:07,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/103766b7fa96423eb53998f9c1f5ca1f, entries=150, sequenceid=201, filesize=11.9 K 2024-11-15T22:17:07,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 94761791c1172512c89bb88a26d6fda0 in 908ms, sequenceid=201, compaction requested=true 2024-11-15T22:17:07,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:07,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:07,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:07,185 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:07,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:07,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:07,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:07,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:17:07,188 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:07,203 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:07,204 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/A is initiating minor compaction (all files) 2024-11-15T22:17:07,204 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/A in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:07,204 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/7e9ce2a5107a41aba83e970e9d7c4e52, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/9d70cfdc6040447bb0536eb3f3deac0d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d566dae6d00040bd9b01189a6a909393] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=91.5 K 2024-11-15T22:17:07,204 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:07,204 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/7e9ce2a5107a41aba83e970e9d7c4e52, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/9d70cfdc6040447bb0536eb3f3deac0d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d566dae6d00040bd9b01189a6a909393] 2024-11-15T22:17:07,204 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:07,204 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/B is initiating minor compaction (all files) 2024-11-15T22:17:07,204 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/B in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:07,204 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/d07a0a36756247999bfe28dfe80561d7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/1c8fc7c313fb4f98a6f09ac56d3352b6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/8d4969b0467e4814b2acc0107e526fa8] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=35.9 K 2024-11-15T22:17:07,208 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e9ce2a5107a41aba83e970e9d7c4e52, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731709022765 2024-11-15T22:17:07,208 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d07a0a36756247999bfe28dfe80561d7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731709022765 2024-11-15T22:17:07,212 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c8fc7c313fb4f98a6f09ac56d3352b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731709022830 2024-11-15T22:17:07,212 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d70cfdc6040447bb0536eb3f3deac0d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731709022830 2024-11-15T22:17:07,215 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d4969b0467e4814b2acc0107e526fa8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731709024069 2024-11-15T22:17:07,220 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting d566dae6d00040bd9b01189a6a909393, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731709024069 2024-11-15T22:17:07,263 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#B#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:07,264 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/497cb6512ff74dc3b36f985b8e8d2ed6 is 50, key is test_row_0/B:col10/1731709024069/Put/seqid=0 2024-11-15T22:17:07,266 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T22:17:07,271 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:07,272 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:17:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-15T22:17:07,277 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:17:07,278 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:17:07,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:17:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T22:17:07,293 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115b9e52c81334a43879230a6b0624d596d_94761791c1172512c89bb88a26d6fda0 store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:07,295 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115b9e52c81334a43879230a6b0624d596d_94761791c1172512c89bb88a26d6fda0, store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:07,295 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b9e52c81334a43879230a6b0624d596d_94761791c1172512c89bb88a26d6fda0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:07,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742202_1378 (size=12595) 2024-11-15T22:17:07,353 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/497cb6512ff74dc3b36f985b8e8d2ed6 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/497cb6512ff74dc3b36f985b8e8d2ed6 2024-11-15T22:17:07,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742203_1379 (size=4469) 2024-11-15T22:17:07,361 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#A#compaction#319 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:07,362 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/936d95074be6497cb39ec7bbd9dffc53 is 175, key is test_row_0/A:col10/1731709024069/Put/seqid=0 2024-11-15T22:17:07,371 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/B of 94761791c1172512c89bb88a26d6fda0 into 497cb6512ff74dc3b36f985b8e8d2ed6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:07,371 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:07,371 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/B, priority=13, startTime=1731709027185; duration=0sec 2024-11-15T22:17:07,371 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:07,372 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:B 2024-11-15T22:17:07,372 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:07,373 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:07,373 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/C is initiating minor compaction (all files) 2024-11-15T22:17:07,373 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/C in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:07,374 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/8f06802e78e94dbfbdbd31172f149a7c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/8f00194263ca414682d9446cac5b3574, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/103766b7fa96423eb53998f9c1f5ca1f] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=35.9 K 2024-11-15T22:17:07,374 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f06802e78e94dbfbdbd31172f149a7c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731709022765 2024-11-15T22:17:07,374 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f00194263ca414682d9446cac5b3574, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731709022830 2024-11-15T22:17:07,375 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 103766b7fa96423eb53998f9c1f5ca1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731709024069 2024-11-15T22:17:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T22:17:07,393 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#C#compaction#320 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:07,394 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/515e1d8cf5d94b4ebd67cf0d250255af is 50, key is test_row_0/C:col10/1731709024069/Put/seqid=0 2024-11-15T22:17:07,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742204_1380 (size=31549) 2024-11-15T22:17:07,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T22:17:07,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:07,437 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T22:17:07,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:17:07,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:07,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:17:07,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:07,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:17:07,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:07,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742205_1381 (size=12595) 2024-11-15T22:17:07,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111581fdbf8d0a904a09bc983715ce6d0409_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709026353/Put/seqid=0 2024-11-15T22:17:07,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742206_1382 (size=12304) 2024-11-15T22:17:07,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:07,547 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111581fdbf8d0a904a09bc983715ce6d0409_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111581fdbf8d0a904a09bc983715ce6d0409_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:07,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:07,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/77a440bc49ec48ee98389fe764ed261a, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:07,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/77a440bc49ec48ee98389fe764ed261a is 175, key is test_row_0/A:col10/1731709026353/Put/seqid=0 2024-11-15T22:17:07,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742207_1383 (size=31105) 2024-11-15T22:17:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T22:17:07,605 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/77a440bc49ec48ee98389fe764ed261a 2024-11-15T22:17:07,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/a248d6b85e584bb8954503234458e14d is 50, key is test_row_0/B:col10/1731709026353/Put/seqid=0 2024-11-15T22:17:07,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:07,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709087668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:07,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:07,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709087669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:07,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709087685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:07,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709087687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:07,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742208_1384 (size=12151) 2024-11-15T22:17:07,740 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/a248d6b85e584bb8954503234458e14d 2024-11-15T22:17:07,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/7128351b1ce9426d81b135ab07bbd971 is 50, key is test_row_0/C:col10/1731709026353/Put/seqid=0 2024-11-15T22:17:07,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742209_1385 (size=12151) 2024-11-15T22:17:07,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:07,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:07,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709087823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:07,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709087823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:07,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:07,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709087823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:07,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:07,843 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/7128351b1ce9426d81b135ab07bbd971 2024-11-15T22:17:07,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709087825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:07,862 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/936d95074be6497cb39ec7bbd9dffc53 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/936d95074be6497cb39ec7bbd9dffc53 2024-11-15T22:17:07,877 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/A of 94761791c1172512c89bb88a26d6fda0 into 936d95074be6497cb39ec7bbd9dffc53(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:07,877 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:07,877 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/A, priority=13, startTime=1731709027184; duration=0sec 2024-11-15T22:17:07,877 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:07,877 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:A 2024-11-15T22:17:07,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/77a440bc49ec48ee98389fe764ed261a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/77a440bc49ec48ee98389fe764ed261a 2024-11-15T22:17:07,908 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/515e1d8cf5d94b4ebd67cf0d250255af as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/515e1d8cf5d94b4ebd67cf0d250255af 2024-11-15T22:17:07,912 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/77a440bc49ec48ee98389fe764ed261a, entries=150, sequenceid=220, filesize=30.4 K 2024-11-15T22:17:07,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T22:17:07,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/a248d6b85e584bb8954503234458e14d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/a248d6b85e584bb8954503234458e14d 2024-11-15T22:17:07,929 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/C of 94761791c1172512c89bb88a26d6fda0 into 515e1d8cf5d94b4ebd67cf0d250255af(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:07,929 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:07,929 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/C, priority=13, startTime=1731709027185; duration=0sec 2024-11-15T22:17:07,930 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:07,930 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:C 2024-11-15T22:17:07,945 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/a248d6b85e584bb8954503234458e14d, entries=150, sequenceid=220, filesize=11.9 K 2024-11-15T22:17:07,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/7128351b1ce9426d81b135ab07bbd971 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/7128351b1ce9426d81b135ab07bbd971 2024-11-15T22:17:07,997 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/7128351b1ce9426d81b135ab07bbd971, entries=150, sequenceid=220, filesize=11.9 K 2024-11-15T22:17:08,001 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 94761791c1172512c89bb88a26d6fda0 in 565ms, sequenceid=220, compaction requested=false 2024-11-15T22:17:08,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:08,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:08,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-15T22:17:08,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-15T22:17:08,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-15T22:17:08,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 727 msec 2024-11-15T22:17:08,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 735 msec 2024-11-15T22:17:08,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:08,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T22:17:08,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:17:08,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:08,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:17:08,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:08,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:17:08,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:08,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159945f141bf2f4227b948367e406a87bf_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709028056/Put/seqid=0 2024-11-15T22:17:08,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742210_1386 (size=17284) 2024-11-15T22:17:08,136 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:08,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709088137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,155 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159945f141bf2f4227b948367e406a87bf_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159945f141bf2f4227b948367e406a87bf_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:08,157 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/134fcbc7726c4251802d3a7acee8ec5f, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:08,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/134fcbc7726c4251802d3a7acee8ec5f is 175, key is test_row_0/A:col10/1731709028056/Put/seqid=0 2024-11-15T22:17:08,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709088145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709088153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709088155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742211_1387 (size=48389) 2024-11-15T22:17:08,244 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/134fcbc7726c4251802d3a7acee8ec5f 2024-11-15T22:17:08,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/b401f4e10ff74006b579426deaa32c64 is 50, key is test_row_0/B:col10/1731709028056/Put/seqid=0 2024-11-15T22:17:08,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709088264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709088271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709088291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742212_1388 (size=12151) 2024-11-15T22:17:08,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/b401f4e10ff74006b579426deaa32c64 2024-11-15T22:17:08,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/983649c67a7b4640999f7ba84f92449a is 50, key is test_row_0/C:col10/1731709028056/Put/seqid=0 2024-11-15T22:17:08,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709088301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742213_1389 (size=12151) 2024-11-15T22:17:08,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/983649c67a7b4640999f7ba84f92449a 2024-11-15T22:17:08,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/134fcbc7726c4251802d3a7acee8ec5f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/134fcbc7726c4251802d3a7acee8ec5f 2024-11-15T22:17:08,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/134fcbc7726c4251802d3a7acee8ec5f, entries=250, sequenceid=238, filesize=47.3 K 2024-11-15T22:17:08,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/b401f4e10ff74006b579426deaa32c64 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/b401f4e10ff74006b579426deaa32c64 2024-11-15T22:17:08,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/b401f4e10ff74006b579426deaa32c64, entries=150, sequenceid=238, filesize=11.9 K 2024-11-15T22:17:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T22:17:08,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/983649c67a7b4640999f7ba84f92449a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/983649c67a7b4640999f7ba84f92449a 2024-11-15T22:17:08,428 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:08,429 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:17:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-15T22:17:08,434 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:17:08,435 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:17:08,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:17:08,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T22:17:08,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/983649c67a7b4640999f7ba84f92449a, entries=150, sequenceid=238, filesize=11.9 K 2024-11-15T22:17:08,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 94761791c1172512c89bb88a26d6fda0 in 421ms, sequenceid=238, compaction requested=true 2024-11-15T22:17:08,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:08,488 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:08,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:08,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:08,489 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:08,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:08,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:08,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:08,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:08,508 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:08,508 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/A is initiating minor compaction (all files) 2024-11-15T22:17:08,508 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/A in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:08,508 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/936d95074be6497cb39ec7bbd9dffc53, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/77a440bc49ec48ee98389fe764ed261a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/134fcbc7726c4251802d3a7acee8ec5f] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=108.4 K 2024-11-15T22:17:08,509 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:08,509 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/936d95074be6497cb39ec7bbd9dffc53, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/77a440bc49ec48ee98389fe764ed261a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/134fcbc7726c4251802d3a7acee8ec5f] 2024-11-15T22:17:08,509 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:08,509 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/B is initiating minor compaction (all files) 2024-11-15T22:17:08,509 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/B in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:08,509 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/497cb6512ff74dc3b36f985b8e8d2ed6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/a248d6b85e584bb8954503234458e14d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/b401f4e10ff74006b579426deaa32c64] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=36.0 K 2024-11-15T22:17:08,511 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 936d95074be6497cb39ec7bbd9dffc53, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731709024069 2024-11-15T22:17:08,514 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 497cb6512ff74dc3b36f985b8e8d2ed6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731709024069 2024-11-15T22:17:08,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:08,516 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting a248d6b85e584bb8954503234458e14d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731709026329 2024-11-15T22:17:08,517 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77a440bc49ec48ee98389fe764ed261a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731709026329 2024-11-15T22:17:08,517 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting b401f4e10ff74006b579426deaa32c64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731709027607 2024-11-15T22:17:08,517 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 134fcbc7726c4251802d3a7acee8ec5f, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731709027607 2024-11-15T22:17:08,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T22:17:08,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:17:08,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:08,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:17:08,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:08,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:17:08,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:08,541 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:08,542 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/536517ab0ac64e6f8550f965f12bade3 is 50, key is test_row_0/B:col10/1731709028056/Put/seqid=0 2024-11-15T22:17:08,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T22:17:08,575 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:08,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159e9360a5cf544a97a8b20ce88e0cb631_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709028511/Put/seqid=0 2024-11-15T22:17:08,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742214_1390 (size=12697) 2024-11-15T22:17:08,583 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115cb7dc04537014d1981511f7ce6bdb4eb_94761791c1172512c89bb88a26d6fda0 store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:08,585 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115cb7dc04537014d1981511f7ce6bdb4eb_94761791c1172512c89bb88a26d6fda0, store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:08,585 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cb7dc04537014d1981511f7ce6bdb4eb_94761791c1172512c89bb88a26d6fda0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:08,594 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/536517ab0ac64e6f8550f965f12bade3 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/536517ab0ac64e6f8550f965f12bade3 2024-11-15T22:17:08,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T22:17:08,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:08,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:08,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:08,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:08,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:08,600 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/B of 94761791c1172512c89bb88a26d6fda0 into 536517ab0ac64e6f8550f965f12bade3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:08,600 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:08,600 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/B, priority=13, startTime=1731709028489; duration=0sec 2024-11-15T22:17:08,600 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:08,600 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:B 2024-11-15T22:17:08,601 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:08,607 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:08,607 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/C is initiating minor compaction (all files) 2024-11-15T22:17:08,607 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/C in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:08,608 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/515e1d8cf5d94b4ebd67cf0d250255af, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/7128351b1ce9426d81b135ab07bbd971, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/983649c67a7b4640999f7ba84f92449a] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=36.0 K 2024-11-15T22:17:08,608 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 515e1d8cf5d94b4ebd67cf0d250255af, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731709024069 2024-11-15T22:17:08,609 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 7128351b1ce9426d81b135ab07bbd971, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731709026329 2024-11-15T22:17:08,610 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 983649c67a7b4640999f7ba84f92449a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731709027607 2024-11-15T22:17:08,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709088601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709088607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709088608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709088617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742215_1391 (size=14844) 2024-11-15T22:17:08,650 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#C#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:08,651 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/13e4aadd3f7f41b5b6b85764d414321e is 50, key is test_row_0/C:col10/1731709028056/Put/seqid=0 2024-11-15T22:17:08,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742216_1392 (size=4469) 2024-11-15T22:17:08,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742217_1393 (size=12697) 2024-11-15T22:17:08,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709088732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T22:17:08,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:08,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:08,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709088732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:08,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:08,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:08,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709088733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:08,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T22:17:08,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709088743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,776 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/13e4aadd3f7f41b5b6b85764d414321e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/13e4aadd3f7f41b5b6b85764d414321e 2024-11-15T22:17:08,824 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/C of 94761791c1172512c89bb88a26d6fda0 into 13e4aadd3f7f41b5b6b85764d414321e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:08,824 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:08,824 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/C, priority=13, startTime=1731709028489; duration=0sec 2024-11-15T22:17:08,824 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:08,824 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:C 2024-11-15T22:17:08,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T22:17:08,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:08,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:08,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:08,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:08,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:08,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709088962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709088962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709088971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709088972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:09,051 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:09,060 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#A#compaction#329 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:09,061 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/be60e0bec061414191b63b93b9affdf3 is 175, key is test_row_0/A:col10/1731709028056/Put/seqid=0 2024-11-15T22:17:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T22:17:09,079 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159e9360a5cf544a97a8b20ce88e0cb631_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159e9360a5cf544a97a8b20ce88e0cb631_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:09,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T22:17:09,085 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/d5822703e40742ee9cfb686497049b6f, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:09,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:09,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:09,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:09,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/d5822703e40742ee9cfb686497049b6f is 175, key is test_row_0/A:col10/1731709028511/Put/seqid=0 2024-11-15T22:17:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742218_1394 (size=31651) 2024-11-15T22:17:09,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742219_1395 (size=39799) 2024-11-15T22:17:09,121 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/d5822703e40742ee9cfb686497049b6f 2024-11-15T22:17:09,145 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/be60e0bec061414191b63b93b9affdf3 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/be60e0bec061414191b63b93b9affdf3 2024-11-15T22:17:09,175 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/A of 94761791c1172512c89bb88a26d6fda0 into be60e0bec061414191b63b93b9affdf3(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:09,175 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:09,175 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/A, priority=13, startTime=1731709028488; duration=0sec 2024-11-15T22:17:09,175 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:09,175 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:A 2024-11-15T22:17:09,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/3614c1678dee4ad981a62bdc1e931f86 is 50, key is test_row_0/B:col10/1731709028511/Put/seqid=0 2024-11-15T22:17:09,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742220_1396 (size=12201) 2024-11-15T22:17:09,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/3614c1678dee4ad981a62bdc1e931f86 2024-11-15T22:17:09,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T22:17:09,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:09,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:09,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:09,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/24ac16e02b0f4175920c93c6b017c76d is 50, key is test_row_0/C:col10/1731709028511/Put/seqid=0 2024-11-15T22:17:09,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709089282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709089284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:09,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709089291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:09,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:09,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709089291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:09,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:09,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742221_1397 (size=12201) 2024-11-15T22:17:09,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T22:17:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:09,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T22:17:09,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:09,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:09,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:09,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T22:17:09,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/24ac16e02b0f4175920c93c6b017c76d 2024-11-15T22:17:09,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T22:17:09,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:09,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:09,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:09,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/d5822703e40742ee9cfb686497049b6f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d5822703e40742ee9cfb686497049b6f 2024-11-15T22:17:09,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d5822703e40742ee9cfb686497049b6f, entries=200, sequenceid=257, filesize=38.9 K 2024-11-15T22:17:09,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/3614c1678dee4ad981a62bdc1e931f86 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/3614c1678dee4ad981a62bdc1e931f86 2024-11-15T22:17:09,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/3614c1678dee4ad981a62bdc1e931f86, entries=150, sequenceid=257, filesize=11.9 K 2024-11-15T22:17:09,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/24ac16e02b0f4175920c93c6b017c76d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/24ac16e02b0f4175920c93c6b017c76d 2024-11-15T22:17:09,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/24ac16e02b0f4175920c93c6b017c76d, entries=150, sequenceid=257, filesize=11.9 K 2024-11-15T22:17:09,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 94761791c1172512c89bb88a26d6fda0 in 1280ms, sequenceid=257, compaction requested=false 2024-11-15T22:17:09,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:09,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T22:17:09,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:17:09,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:09,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:17:09,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:09,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:17:09,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:09,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115847ce931ad624d948e331bf012e9574f_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709029817/Put/seqid=0 2024-11-15T22:17:09,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742222_1398 (size=17534) 2024-11-15T22:17:09,885 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:09,890 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115847ce931ad624d948e331bf012e9574f_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115847ce931ad624d948e331bf012e9574f_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:09,893 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/97942f56e8ef4ec78708611cf0a3f3cf, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:09,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/97942f56e8ef4ec78708611cf0a3f3cf is 175, key is test_row_0/A:col10/1731709029817/Put/seqid=0 2024-11-15T22:17:09,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T22:17:09,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:09,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:09,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:09,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742223_1399 (size=48639) 2024-11-15T22:17:09,910 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/97942f56e8ef4ec78708611cf0a3f3cf 2024-11-15T22:17:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:09,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:09,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709089901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:09,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:09,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709089901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:09,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:09,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:09,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709089911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:09,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709089913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:09,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:09,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/6e19f4590c704521b602bb7ac5590961 is 50, key is test_row_0/B:col10/1731709029817/Put/seqid=0 2024-11-15T22:17:09,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742224_1400 (size=12301) 2024-11-15T22:17:10,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709090031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709090031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:10,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709090041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:10,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709090042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:10,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T22:17:10,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:10,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:10,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:10,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:10,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:10,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T22:17:10,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:10,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:10,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:10,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:10,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:10,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709090265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:10,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709090265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:10,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709090269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:10,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709090271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:10,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/6e19f4590c704521b602bb7ac5590961 2024-11-15T22:17:10,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/67febe58ce69411491c27256c9f32c20 is 50, key is test_row_0/C:col10/1731709029817/Put/seqid=0 2024-11-15T22:17:10,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T22:17:10,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:10,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:10,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:10,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:10,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:10,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742225_1401 (size=12301) 2024-11-15T22:17:10,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/67febe58ce69411491c27256c9f32c20 2024-11-15T22:17:10,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/97942f56e8ef4ec78708611cf0a3f3cf as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/97942f56e8ef4ec78708611cf0a3f3cf 2024-11-15T22:17:10,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/97942f56e8ef4ec78708611cf0a3f3cf, entries=250, sequenceid=276, filesize=47.5 K 2024-11-15T22:17:10,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/6e19f4590c704521b602bb7ac5590961 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/6e19f4590c704521b602bb7ac5590961 2024-11-15T22:17:10,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/6e19f4590c704521b602bb7ac5590961, entries=150, sequenceid=276, filesize=12.0 K 2024-11-15T22:17:10,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/67febe58ce69411491c27256c9f32c20 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/67febe58ce69411491c27256c9f32c20 2024-11-15T22:17:10,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T22:17:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:10,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T22:17:10,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709090583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:10,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709090583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:10,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709090583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:10,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709090584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:10,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:10,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:10,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/67febe58ce69411491c27256c9f32c20, entries=150, sequenceid=276, filesize=12.0 K 2024-11-15T22:17:10,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 94761791c1172512c89bb88a26d6fda0 in 789ms, sequenceid=276, compaction requested=true 2024-11-15T22:17:10,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:10,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:10,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:10,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:10,607 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:10,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:10,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:10,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:17:10,607 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:10,621 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:10,621 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/B is initiating minor compaction (all files) 2024-11-15T22:17:10,621 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/B in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:10,621 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/536517ab0ac64e6f8550f965f12bade3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/3614c1678dee4ad981a62bdc1e931f86, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/6e19f4590c704521b602bb7ac5590961] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=36.3 K 2024-11-15T22:17:10,625 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120089 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:10,625 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/A is initiating minor compaction (all files) 2024-11-15T22:17:10,625 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/A in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:10,625 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/be60e0bec061414191b63b93b9affdf3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d5822703e40742ee9cfb686497049b6f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/97942f56e8ef4ec78708611cf0a3f3cf] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=117.3 K 2024-11-15T22:17:10,625 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:10,625 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/be60e0bec061414191b63b93b9affdf3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d5822703e40742ee9cfb686497049b6f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/97942f56e8ef4ec78708611cf0a3f3cf] 2024-11-15T22:17:10,626 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 536517ab0ac64e6f8550f965f12bade3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731709027607 2024-11-15T22:17:10,628 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting be60e0bec061414191b63b93b9affdf3, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731709027607 2024-11-15T22:17:10,632 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3614c1678dee4ad981a62bdc1e931f86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731709028148 2024-11-15T22:17:10,633 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e19f4590c704521b602bb7ac5590961, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731709028596 2024-11-15T22:17:10,633 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d5822703e40742ee9cfb686497049b6f, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731709028101 2024-11-15T22:17:10,634 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 97942f56e8ef4ec78708611cf0a3f3cf, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731709028558 2024-11-15T22:17:10,687 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#B#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:10,687 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/175cf533f5ae463b8e44662b29abd225 is 50, key is test_row_0/B:col10/1731709029817/Put/seqid=0 2024-11-15T22:17:10,689 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:10,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T22:17:10,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:10,728 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T22:17:10,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:17:10,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:10,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:17:10,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:10,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:17:10,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:10,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742226_1402 (size=12949) 2024-11-15T22:17:10,743 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111594cd45983fe54ee2b1a94eb1b6bd03a1_94761791c1172512c89bb88a26d6fda0 store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:10,745 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111594cd45983fe54ee2b1a94eb1b6bd03a1_94761791c1172512c89bb88a26d6fda0, store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:10,746 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111594cd45983fe54ee2b1a94eb1b6bd03a1_94761791c1172512c89bb88a26d6fda0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:10,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156e13751257b04ebfac75d41752d97ad9_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709029894/Put/seqid=0 2024-11-15T22:17:10,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742227_1403 (size=4469) 2024-11-15T22:17:10,845 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#A#compaction#337 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:10,846 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/77cf39c13b9241deb845af5886d5f8e1 is 175, key is test_row_0/A:col10/1731709029817/Put/seqid=0 2024-11-15T22:17:10,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742228_1404 (size=12454) 2024-11-15T22:17:10,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:10,882 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156e13751257b04ebfac75d41752d97ad9_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156e13751257b04ebfac75d41752d97ad9_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/2964516225934b708a308699f0fe1dff, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/2964516225934b708a308699f0fe1dff is 175, key is test_row_0/A:col10/1731709029894/Put/seqid=0 2024-11-15T22:17:10,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742229_1405 (size=31903) 2024-11-15T22:17:10,933 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/77cf39c13b9241deb845af5886d5f8e1 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/77cf39c13b9241deb845af5886d5f8e1 2024-11-15T22:17:10,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742230_1406 (size=31255) 2024-11-15T22:17:10,948 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/A of 94761791c1172512c89bb88a26d6fda0 into 77cf39c13b9241deb845af5886d5f8e1(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:10,948 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:10,948 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/A, priority=13, startTime=1731709030607; duration=0sec 2024-11-15T22:17:10,948 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:10,949 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:A 2024-11-15T22:17:10,949 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:10,960 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:10,960 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/C is initiating minor compaction (all files) 2024-11-15T22:17:10,961 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/C in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:10,961 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/13e4aadd3f7f41b5b6b85764d414321e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/24ac16e02b0f4175920c93c6b017c76d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/67febe58ce69411491c27256c9f32c20] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=36.3 K 2024-11-15T22:17:10,964 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 13e4aadd3f7f41b5b6b85764d414321e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731709027607 2024-11-15T22:17:10,968 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 24ac16e02b0f4175920c93c6b017c76d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731709028148 2024-11-15T22:17:10,972 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 67febe58ce69411491c27256c9f32c20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731709028596 2024-11-15T22:17:11,028 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#C#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:11,029 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/bc5e7cf4f33c45d5931633b9d9ad6e05 is 50, key is test_row_0/C:col10/1731709029817/Put/seqid=0 2024-11-15T22:17:11,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742231_1407 (size=12949) 2024-11-15T22:17:11,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:11,151 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/bc5e7cf4f33c45d5931633b9d9ad6e05 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/bc5e7cf4f33c45d5931633b9d9ad6e05 2024-11-15T22:17:11,186 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/175cf533f5ae463b8e44662b29abd225 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/175cf533f5ae463b8e44662b29abd225 2024-11-15T22:17:11,187 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/C of 94761791c1172512c89bb88a26d6fda0 into bc5e7cf4f33c45d5931633b9d9ad6e05(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:11,187 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:11,187 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/C, priority=13, startTime=1731709030607; duration=0sec 2024-11-15T22:17:11,187 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:11,187 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:C 2024-11-15T22:17:11,221 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/B of 94761791c1172512c89bb88a26d6fda0 into 175cf533f5ae463b8e44662b29abd225(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:11,221 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:11,221 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/B, priority=13, startTime=1731709030607; duration=0sec 2024-11-15T22:17:11,221 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:11,221 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:B 2024-11-15T22:17:11,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709091227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709091227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709091228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709091242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,345 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/2964516225934b708a308699f0fe1dff 2024-11-15T22:17:11,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709091351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709091353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/9f85e013c9c344cc943d90716b7aed61 is 50, key is test_row_0/B:col10/1731709029894/Put/seqid=0 2024-11-15T22:17:11,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709091371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709091372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742232_1408 (size=12301) 2024-11-15T22:17:11,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709091564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709091572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709091593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709091600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,823 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/9f85e013c9c344cc943d90716b7aed61 2024-11-15T22:17:11,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/7dc8ecc601524b268159ea378bf52710 is 50, key is test_row_0/C:col10/1731709029894/Put/seqid=0 2024-11-15T22:17:11,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709091892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709091893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709091911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709091933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:11,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742233_1409 (size=12301) 2024-11-15T22:17:12,356 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/7dc8ecc601524b268159ea378bf52710 2024-11-15T22:17:12,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/2964516225934b708a308699f0fe1dff as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/2964516225934b708a308699f0fe1dff 2024-11-15T22:17:12,411 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/2964516225934b708a308699f0fe1dff, entries=150, sequenceid=292, filesize=30.5 K 2024-11-15T22:17:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/9f85e013c9c344cc943d90716b7aed61 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/9f85e013c9c344cc943d90716b7aed61 2024-11-15T22:17:12,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709092412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:12,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709092422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:12,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:12,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:12,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:12,450 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/9f85e013c9c344cc943d90716b7aed61, entries=150, sequenceid=292, filesize=12.0 K 2024-11-15T22:17:12,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709092442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:12,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/7dc8ecc601524b268159ea378bf52710 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/7dc8ecc601524b268159ea378bf52710 2024-11-15T22:17:12,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:12,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:12,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:12,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709092442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:12,478 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/7dc8ecc601524b268159ea378bf52710, entries=150, sequenceid=292, filesize=12.0 K 2024-11-15T22:17:12,483 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 94761791c1172512c89bb88a26d6fda0 in 1754ms, sequenceid=292, compaction requested=false 2024-11-15T22:17:12,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:12,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:12,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-15T22:17:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-15T22:17:12,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-15T22:17:12,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.0500 sec 2024-11-15T22:17:12,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 4.0590 sec 2024-11-15T22:17:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T22:17:12,614 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:12,619 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:17:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-15T22:17:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T22:17:12,632 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:17:12,637 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:17:12,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:17:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T22:17:12,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T22:17:12,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:12,801 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T22:17:12,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:17:12,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:12,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:17:12,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:12,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:17:12,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:12,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b5c4b00c9e094484be7d81d8b08c7cb7_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709031228/Put/seqid=0 2024-11-15T22:17:12,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742234_1410 (size=12454) 2024-11-15T22:17:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T22:17:13,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:13,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T22:17:13,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:13,276 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b5c4b00c9e094484be7d81d8b08c7cb7_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b5c4b00c9e094484be7d81d8b08c7cb7_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:13,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/725c1295e62e4325a6dc1761cf0a1a8a, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:13,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/725c1295e62e4325a6dc1761cf0a1a8a is 175, key is test_row_0/A:col10/1731709031228/Put/seqid=0 2024-11-15T22:17:13,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742235_1411 (size=31255) 2024-11-15T22:17:13,307 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/725c1295e62e4325a6dc1761cf0a1a8a 2024-11-15T22:17:13,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/2d89b3ad336c458bba40631b9a59bd29 is 50, key is test_row_0/B:col10/1731709031228/Put/seqid=0 2024-11-15T22:17:13,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742236_1412 (size=12301) 2024-11-15T22:17:13,380 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/2d89b3ad336c458bba40631b9a59bd29 2024-11-15T22:17:13,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58448 deadline: 1731709093370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:13,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/183774bcb4804ff59d3810fa29c890fe is 50, key is test_row_0/C:col10/1731709031228/Put/seqid=0 2024-11-15T22:17:13,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58516 deadline: 1731709093445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:13,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58454 deadline: 1731709093446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:13,457 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in 94761791c1172512c89bb88a26d6fda0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:17:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:13,458 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in 94761791c1172512c89bb88a26d6fda0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:17:13,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:13,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:13,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:13,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742237_1413 (size=12301) 2024-11-15T22:17:13,488 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/183774bcb4804ff59d3810fa29c890fe 2024-11-15T22:17:13,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731709093464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:13,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58512 deadline: 1731709093467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:13,491 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in 94761791c1172512c89bb88a26d6fda0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:17:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:13,493 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_0' in 94761791c1172512c89bb88a26d6fda0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:17:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:13,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58448 deadline: 1731709093502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:13,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:13,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:13,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:13,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/725c1295e62e4325a6dc1761cf0a1a8a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/725c1295e62e4325a6dc1761cf0a1a8a 2024-11-15T22:17:13,541 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/725c1295e62e4325a6dc1761cf0a1a8a, entries=150, sequenceid=311, filesize=30.5 K 2024-11-15T22:17:13,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/2d89b3ad336c458bba40631b9a59bd29 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/2d89b3ad336c458bba40631b9a59bd29 2024-11-15T22:17:13,583 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/2d89b3ad336c458bba40631b9a59bd29, entries=150, sequenceid=311, filesize=12.0 K 2024-11-15T22:17:13,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/183774bcb4804ff59d3810fa29c890fe as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/183774bcb4804ff59d3810fa29c890fe 2024-11-15T22:17:13,622 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/183774bcb4804ff59d3810fa29c890fe, entries=150, sequenceid=311, filesize=12.0 K 2024-11-15T22:17:13,628 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 94761791c1172512c89bb88a26d6fda0 in 827ms, sequenceid=311, compaction requested=true 2024-11-15T22:17:13,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:13,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:13,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-15T22:17:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-15T22:17:13,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-15T22:17:13,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0110 sec 2024-11-15T22:17:13,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.0590 sec 2024-11-15T22:17:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:13,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T22:17:13,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:17:13,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:13,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:17:13,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:13,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:17:13,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:13,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115df63a895f19c4280b52ad2397985a392_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709033328/Put/seqid=0 2024-11-15T22:17:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T22:17:13,777 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:13,779 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:17:13,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-15T22:17:13,785 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:17:13,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T22:17:13,786 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:17:13,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:17:13,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742238_1414 (size=14994) 2024-11-15T22:17:13,836 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:13,867 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115df63a895f19c4280b52ad2397985a392_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115df63a895f19c4280b52ad2397985a392_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:13,878 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/f2b56d6b73c34a9eb4f9ee23fb31134a, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:13,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/f2b56d6b73c34a9eb4f9ee23fb31134a is 175, key is test_row_0/A:col10/1731709033328/Put/seqid=0 2024-11-15T22:17:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T22:17:13,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742239_1415 (size=39949) 2024-11-15T22:17:13,915 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/f2b56d6b73c34a9eb4f9ee23fb31134a 2024-11-15T22:17:13,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/e82b1fc81e9d4929bddca541813df5aa is 50, key is test_row_0/B:col10/1731709033328/Put/seqid=0 2024-11-15T22:17:13,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T22:17:13,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:13,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:13,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:13,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:13,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:13,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742240_1416 (size=12301) 2024-11-15T22:17:13,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/e82b1fc81e9d4929bddca541813df5aa 2024-11-15T22:17:13,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58448 deadline: 1731709093964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:13,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:13,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:13,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:14,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/a5ace5be68d84cf49f14b51d462d67b2 is 50, key is test_row_0/C:col10/1731709033328/Put/seqid=0 2024-11-15T22:17:14,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742241_1417 (size=12301) 2024-11-15T22:17:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T22:17:14,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/a5ace5be68d84cf49f14b51d462d67b2 2024-11-15T22:17:14,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58448 deadline: 1731709094096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:14,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T22:17:14,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:14,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:14,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:14,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/f2b56d6b73c34a9eb4f9ee23fb31134a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/f2b56d6b73c34a9eb4f9ee23fb31134a 2024-11-15T22:17:14,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:14,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:14,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/f2b56d6b73c34a9eb4f9ee23fb31134a, entries=200, sequenceid=328, filesize=39.0 K 2024-11-15T22:17:14,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/e82b1fc81e9d4929bddca541813df5aa as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/e82b1fc81e9d4929bddca541813df5aa 2024-11-15T22:17:14,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/e82b1fc81e9d4929bddca541813df5aa, entries=150, sequenceid=328, filesize=12.0 K 2024-11-15T22:17:14,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/a5ace5be68d84cf49f14b51d462d67b2 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/a5ace5be68d84cf49f14b51d462d67b2 2024-11-15T22:17:14,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/a5ace5be68d84cf49f14b51d462d67b2, entries=150, sequenceid=328, filesize=12.0 K 2024-11-15T22:17:14,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 94761791c1172512c89bb88a26d6fda0 in 393ms, sequenceid=328, compaction requested=true 2024-11-15T22:17:14,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:14,140 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:17:14,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:14,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:14,140 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:17:14,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:14,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:14,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:14,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:14,142 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:17:14,142 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/B is initiating minor compaction (all files) 2024-11-15T22:17:14,142 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/B in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:14,142 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/175cf533f5ae463b8e44662b29abd225, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/9f85e013c9c344cc943d90716b7aed61, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/2d89b3ad336c458bba40631b9a59bd29, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/e82b1fc81e9d4929bddca541813df5aa] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=48.7 K 2024-11-15T22:17:14,142 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:17:14,142 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/A is initiating minor compaction (all files) 2024-11-15T22:17:14,142 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/A in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:14,142 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/77cf39c13b9241deb845af5886d5f8e1, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/2964516225934b708a308699f0fe1dff, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/725c1295e62e4325a6dc1761cf0a1a8a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/f2b56d6b73c34a9eb4f9ee23fb31134a] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=131.2 K 2024-11-15T22:17:14,142 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:14,142 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/77cf39c13b9241deb845af5886d5f8e1, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/2964516225934b708a308699f0fe1dff, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/725c1295e62e4325a6dc1761cf0a1a8a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/f2b56d6b73c34a9eb4f9ee23fb31134a] 2024-11-15T22:17:14,143 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 175cf533f5ae463b8e44662b29abd225, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731709028596 2024-11-15T22:17:14,143 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77cf39c13b9241deb845af5886d5f8e1, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731709028596 2024-11-15T22:17:14,143 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f85e013c9c344cc943d90716b7aed61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731709029894 2024-11-15T22:17:14,144 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2964516225934b708a308699f0fe1dff, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731709029894 2024-11-15T22:17:14,144 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d89b3ad336c458bba40631b9a59bd29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731709031224 2024-11-15T22:17:14,144 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 725c1295e62e4325a6dc1761cf0a1a8a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731709031224 2024-11-15T22:17:14,144 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting e82b1fc81e9d4929bddca541813df5aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731709033328 2024-11-15T22:17:14,145 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2b56d6b73c34a9eb4f9ee23fb31134a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731709033328 2024-11-15T22:17:14,158 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#B#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:14,159 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/0eea00bdf0114f0f892d1280ab8eb11e is 50, key is test_row_0/B:col10/1731709033328/Put/seqid=0 2024-11-15T22:17:14,189 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:14,213 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115acf1882edf264beeb3c58523adc92984_94761791c1172512c89bb88a26d6fda0 store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:14,215 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115acf1882edf264beeb3c58523adc92984_94761791c1172512c89bb88a26d6fda0, store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:14,216 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115acf1882edf264beeb3c58523adc92984_94761791c1172512c89bb88a26d6fda0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:14,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742242_1418 (size=13085) 2024-11-15T22:17:14,227 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/0eea00bdf0114f0f892d1280ab8eb11e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/0eea00bdf0114f0f892d1280ab8eb11e 2024-11-15T22:17:14,239 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/B of 94761791c1172512c89bb88a26d6fda0 into 0eea00bdf0114f0f892d1280ab8eb11e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:14,239 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:14,240 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/B, priority=12, startTime=1731709034140; duration=0sec 2024-11-15T22:17:14,240 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:14,240 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:B 2024-11-15T22:17:14,240 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:17:14,241 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:17:14,241 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 94761791c1172512c89bb88a26d6fda0/C is initiating minor compaction (all files) 2024-11-15T22:17:14,241 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94761791c1172512c89bb88a26d6fda0/C in TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:14,241 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/bc5e7cf4f33c45d5931633b9d9ad6e05, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/7dc8ecc601524b268159ea378bf52710, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/183774bcb4804ff59d3810fa29c890fe, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/a5ace5be68d84cf49f14b51d462d67b2] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp, totalSize=48.7 K 2024-11-15T22:17:14,242 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting bc5e7cf4f33c45d5931633b9d9ad6e05, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731709028596 2024-11-15T22:17:14,242 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dc8ecc601524b268159ea378bf52710, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731709029894 2024-11-15T22:17:14,242 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 183774bcb4804ff59d3810fa29c890fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731709031224 2024-11-15T22:17:14,242 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting a5ace5be68d84cf49f14b51d462d67b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731709033328 2024-11-15T22:17:14,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742243_1419 (size=4469) 2024-11-15T22:17:14,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T22:17:14,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:14,274 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T22:17:14,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:17:14,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:14,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:17:14,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:14,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:17:14,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:14,274 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:14,275 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/9936fea1159c4dd5a8ce222ae02a3130 is 50, key is test_row_0/C:col10/1731709033328/Put/seqid=0 2024-11-15T22:17:14,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742244_1420 (size=13085) 2024-11-15T22:17:14,323 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/9936fea1159c4dd5a8ce222ae02a3130 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/9936fea1159c4dd5a8ce222ae02a3130 2024-11-15T22:17:14,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154fc16b9322b44d058d022bf7ce6a07f0_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709033959/Put/seqid=0 2024-11-15T22:17:14,333 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/C of 94761791c1172512c89bb88a26d6fda0 into 9936fea1159c4dd5a8ce222ae02a3130(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:14,333 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:14,333 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/C, priority=12, startTime=1731709034140; duration=0sec 2024-11-15T22:17:14,333 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:14,333 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:C 2024-11-15T22:17:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:14,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. as already flushing 2024-11-15T22:17:14,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742245_1421 (size=12454) 2024-11-15T22:17:14,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:14,404 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154fc16b9322b44d058d022bf7ce6a07f0_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154fc16b9322b44d058d022bf7ce6a07f0_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:14,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/f0ed630f431248aaa8cf8f6bd06c6bc8, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:14,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/f0ed630f431248aaa8cf8f6bd06c6bc8 is 175, key is test_row_0/A:col10/1731709033959/Put/seqid=0 2024-11-15T22:17:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T22:17:14,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742246_1422 (size=31255) 2024-11-15T22:17:14,423 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/f0ed630f431248aaa8cf8f6bd06c6bc8 2024-11-15T22:17:14,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/eb7a903103bf49e0b381e62ac9fe4446 is 50, key is test_row_0/B:col10/1731709033959/Put/seqid=0 2024-11-15T22:17:14,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742247_1423 (size=12301) 2024-11-15T22:17:14,496 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/eb7a903103bf49e0b381e62ac9fe4446 2024-11-15T22:17:14,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/c64cff8dbfb24779b25ca22b75a346d1 is 50, key is test_row_0/C:col10/1731709033959/Put/seqid=0 2024-11-15T22:17:14,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742248_1424 (size=12301) 2024-11-15T22:17:14,650 INFO [Thread-1495 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1495. 2024-11-15T22:17:14,650 DEBUG [Thread-1495 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:14,650 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:14,651 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:14,651 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:14,652 INFO [Thread-1504 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1504. 2024-11-15T22:17:14,652 DEBUG [Thread-1504 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:14,652 DEBUG [Thread-1504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:14,652 DEBUG [Thread-1504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:14,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:14,663 INFO [Thread-1500 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1500. 2024-11-15T22:17:14,663 DEBUG [Thread-1500 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:14,663 DEBUG [Thread-1500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:14,663 DEBUG [Thread-1500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:14,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:14,664 INFO [Thread-1506 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1506. 2024-11-15T22:17:14,665 DEBUG [Thread-1506 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:14,665 DEBUG [Thread-1506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:14,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:14,665 DEBUG [Thread-1506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:14,669 INFO [Thread-1502 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1502. 2024-11-15T22:17:14,669 DEBUG [Thread-1502 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:14,669 DEBUG [Thread-1502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:14,669 DEBUG [Thread-1502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:14,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:14,678 INFO [Thread-1498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1498. 2024-11-15T22:17:14,678 DEBUG [Thread-1498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:14,678 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:14,678 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:14,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:14,685 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94761791c1172512c89bb88a26d6fda0#A#compaction#349 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:14,685 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/ed87e9fdc81a4af19b0fcd531c7f624e is 175, key is test_row_0/A:col10/1731709033328/Put/seqid=0 2024-11-15T22:17:14,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742249_1425 (size=32039) 2024-11-15T22:17:14,745 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/ed87e9fdc81a4af19b0fcd531c7f624e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/ed87e9fdc81a4af19b0fcd531c7f624e 2024-11-15T22:17:14,754 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94761791c1172512c89bb88a26d6fda0/A of 94761791c1172512c89bb88a26d6fda0 into ed87e9fdc81a4af19b0fcd531c7f624e(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:14,754 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:14,754 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., storeName=94761791c1172512c89bb88a26d6fda0/A, priority=12, startTime=1731709034140; duration=0sec 2024-11-15T22:17:14,754 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:14,754 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:A 2024-11-15T22:17:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T22:17:14,989 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/c64cff8dbfb24779b25ca22b75a346d1 2024-11-15T22:17:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/f0ed630f431248aaa8cf8f6bd06c6bc8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/f0ed630f431248aaa8cf8f6bd06c6bc8 2024-11-15T22:17:14,997 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/f0ed630f431248aaa8cf8f6bd06c6bc8, entries=150, sequenceid=344, filesize=30.5 K 2024-11-15T22:17:14,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/eb7a903103bf49e0b381e62ac9fe4446 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/eb7a903103bf49e0b381e62ac9fe4446 2024-11-15T22:17:15,002 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/eb7a903103bf49e0b381e62ac9fe4446, entries=150, sequenceid=344, filesize=12.0 K 2024-11-15T22:17:15,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/c64cff8dbfb24779b25ca22b75a346d1 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/c64cff8dbfb24779b25ca22b75a346d1 2024-11-15T22:17:15,009 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/c64cff8dbfb24779b25ca22b75a346d1, entries=150, sequenceid=344, filesize=12.0 K 2024-11-15T22:17:15,009 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 94761791c1172512c89bb88a26d6fda0 in 735ms, sequenceid=344, compaction requested=false 2024-11-15T22:17:15,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:15,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:15,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-15T22:17:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-15T22:17:15,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-15T22:17:15,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2240 sec 2024-11-15T22:17:15,014 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.2340 sec 2024-11-15T22:17:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:15,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T22:17:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:17:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:17:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:15,479 INFO [Thread-1491 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1491. 2024-11-15T22:17:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:17:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:15,479 DEBUG [Thread-1491 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:15,479 DEBUG [Thread-1491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:15,479 DEBUG [Thread-1491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:15,481 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:15,483 INFO [Thread-1493 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1493. 2024-11-15T22:17:15,483 DEBUG [Thread-1493 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:15,483 DEBUG [Thread-1493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:15,483 DEBUG [Thread-1493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:15,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:15,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155f8020ba07094f19afb5b73b419fcf73_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709034547/Put/seqid=0 2024-11-15T22:17:15,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742250_1426 (size=12454) 2024-11-15T22:17:15,513 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:15,516 INFO [Thread-1489 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1489. 2024-11-15T22:17:15,516 DEBUG [Thread-1489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:15,516 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:15,516 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:15,516 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:15,518 INFO [Thread-1487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1487. 2024-11-15T22:17:15,518 DEBUG [Thread-1487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:15,518 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:15,519 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:15,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:15,533 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155f8020ba07094f19afb5b73b419fcf73_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155f8020ba07094f19afb5b73b419fcf73_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:15,540 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/84892d9c643c487f8e787f318eb3901a, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:15,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/84892d9c643c487f8e787f318eb3901a is 175, key is test_row_0/A:col10/1731709034547/Put/seqid=0 2024-11-15T22:17:15,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742251_1427 (size=31255) 2024-11-15T22:17:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T22:17:15,942 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 885 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2654 rows 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 870 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2610 rows 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 878 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2624 rows 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 896 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2676 rows 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 886 2024-11-15T22:17:15,942 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2651 rows 2024-11-15T22:17:15,942 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:17:15,942 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:17:15,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:15,943 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:15,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:15,943 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T22:17:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T22:17:15,947 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731709035947"}]},"ts":"1731709035947"} 2024-11-15T22:17:15,948 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T22:17:15,948 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T22:17:15,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:17:15,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94761791c1172512c89bb88a26d6fda0, UNASSIGN}] 2024-11-15T22:17:15,952 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94761791c1172512c89bb88a26d6fda0, UNASSIGN 2024-11-15T22:17:15,952 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=94761791c1172512c89bb88a26d6fda0, regionState=CLOSING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:17:15,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94761791c1172512c89bb88a26d6fda0, UNASSIGN because future has completed 2024-11-15T22:17:15,954 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:17:15,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:17:15,967 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/84892d9c643c487f8e787f318eb3901a 2024-11-15T22:17:15,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/19b52fbf8f0943db80f505b60e965845 is 50, key is test_row_0/B:col10/1731709034547/Put/seqid=0 2024-11-15T22:17:15,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742252_1428 (size=12301) 2024-11-15T22:17:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T22:17:16,106 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(122): Close 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:16,106 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:17:16,106 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1722): Closing 94761791c1172512c89bb88a26d6fda0, disabling compactions & flushes 2024-11-15T22:17:16,106 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T22:17:16,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/19b52fbf8f0943db80f505b60e965845 2024-11-15T22:17:16,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/377dac47874f442293818c04e29c375a is 50, key is test_row_0/C:col10/1731709034547/Put/seqid=0 2024-11-15T22:17:16,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742253_1429 (size=12301) 2024-11-15T22:17:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T22:17:16,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/377dac47874f442293818c04e29c375a 2024-11-15T22:17:16,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/84892d9c643c487f8e787f318eb3901a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/84892d9c643c487f8e787f318eb3901a 2024-11-15T22:17:16,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/84892d9c643c487f8e787f318eb3901a, entries=150, sequenceid=364, filesize=30.5 K 2024-11-15T22:17:16,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/19b52fbf8f0943db80f505b60e965845 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/19b52fbf8f0943db80f505b60e965845 2024-11-15T22:17:16,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/19b52fbf8f0943db80f505b60e965845, entries=150, sequenceid=364, filesize=12.0 K 2024-11-15T22:17:16,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/377dac47874f442293818c04e29c375a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/377dac47874f442293818c04e29c375a 2024-11-15T22:17:16,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/377dac47874f442293818c04e29c375a, entries=150, sequenceid=364, filesize=12.0 K 2024-11-15T22:17:16,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=20.13 KB/20610 for 94761791c1172512c89bb88a26d6fda0 in 1367ms, sequenceid=364, compaction requested=true 2024-11-15T22:17:16,845 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:16,845 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:16,845 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. after waiting 0 ms 2024-11-15T22:17:16,845 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:16,845 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(2902): Flushing 94761791c1172512c89bb88a26d6fda0 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-15T22:17:16,846 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=A 2024-11-15T22:17:16,846 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:16,846 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=B 2024-11-15T22:17:16,846 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:16,846 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94761791c1172512c89bb88a26d6fda0, store=C 2024-11-15T22:17:16,846 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:16,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94761791c1172512c89bb88a26d6fda0: 2024-11-15T22:17:16,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:16,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:16,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:16,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:16,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94761791c1172512c89bb88a26d6fda0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:16,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:16,846 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. because compaction request was cancelled 2024-11-15T22:17:16,846 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. because compaction request was cancelled 2024-11-15T22:17:16,846 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:B 2024-11-15T22:17:16,846 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:A 2024-11-15T22:17:16,846 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. because compaction request was cancelled 2024-11-15T22:17:16,846 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94761791c1172512c89bb88a26d6fda0:C 2024-11-15T22:17:16,872 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c2d6c73aec7e4ae4887c3cd5c9a28d64_94761791c1172512c89bb88a26d6fda0 is 50, key is test_row_0/A:col10/1731709035517/Put/seqid=0 2024-11-15T22:17:16,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742254_1430 (size=12454) 2024-11-15T22:17:16,900 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:16,927 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c2d6c73aec7e4ae4887c3cd5c9a28d64_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c2d6c73aec7e4ae4887c3cd5c9a28d64_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:16,933 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/e7a697882e964b758a2ec3570417ed31, store: [table=TestAcidGuarantees family=A region=94761791c1172512c89bb88a26d6fda0] 2024-11-15T22:17:16,933 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/e7a697882e964b758a2ec3570417ed31 is 175, key is test_row_0/A:col10/1731709035517/Put/seqid=0 2024-11-15T22:17:16,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742255_1431 (size=31255) 2024-11-15T22:17:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T22:17:17,346 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/e7a697882e964b758a2ec3570417ed31 2024-11-15T22:17:17,354 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/150589cff6514bb3844385de56181174 is 50, key is test_row_0/B:col10/1731709035517/Put/seqid=0 2024-11-15T22:17:17,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742256_1432 (size=12301) 2024-11-15T22:17:17,765 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/150589cff6514bb3844385de56181174 2024-11-15T22:17:17,771 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/f63c308d07ea466aa1027662435e5824 is 50, key is test_row_0/C:col10/1731709035517/Put/seqid=0 2024-11-15T22:17:17,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742257_1433 (size=12301) 2024-11-15T22:17:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T22:17:18,190 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/f63c308d07ea466aa1027662435e5824 2024-11-15T22:17:18,194 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/A/e7a697882e964b758a2ec3570417ed31 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/e7a697882e964b758a2ec3570417ed31 2024-11-15T22:17:18,200 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/e7a697882e964b758a2ec3570417ed31, entries=150, sequenceid=370, filesize=30.5 K 2024-11-15T22:17:18,201 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/B/150589cff6514bb3844385de56181174 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/150589cff6514bb3844385de56181174 2024-11-15T22:17:18,207 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/150589cff6514bb3844385de56181174, entries=150, sequenceid=370, filesize=12.0 K 2024-11-15T22:17:18,208 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/.tmp/C/f63c308d07ea466aa1027662435e5824 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/f63c308d07ea466aa1027662435e5824 2024-11-15T22:17:18,213 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/f63c308d07ea466aa1027662435e5824, entries=150, sequenceid=370, filesize=12.0 K 2024-11-15T22:17:18,215 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 94761791c1172512c89bb88a26d6fda0 in 1369ms, sequenceid=370, compaction requested=true 2024-11-15T22:17:18,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/3ab77223d5bf426a860effbe14ba018f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/53c631a4ac154d28b33ac92e397b4fbb, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/a4afe824a7bd41cb8be2f98b4aaccdef, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/6369fbc72af74c1896dd9cc0264e84d0, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/7006aa1a138e498bb283ee62b9216137, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/e843f4a864c245ceb36d330bec940524, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/148fdad423924675b7f7b575b8ee3da9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d51788963cda468bae4908fe8a4d9415, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/3fe637632a2945a8851fb933c759c888, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/7e9ce2a5107a41aba83e970e9d7c4e52, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/1cf9ae84262b4982955ab23cd039c8c8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/9d70cfdc6040447bb0536eb3f3deac0d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/936d95074be6497cb39ec7bbd9dffc53, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d566dae6d00040bd9b01189a6a909393, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/77a440bc49ec48ee98389fe764ed261a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/134fcbc7726c4251802d3a7acee8ec5f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/be60e0bec061414191b63b93b9affdf3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d5822703e40742ee9cfb686497049b6f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/97942f56e8ef4ec78708611cf0a3f3cf, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/77cf39c13b9241deb845af5886d5f8e1, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/2964516225934b708a308699f0fe1dff, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/725c1295e62e4325a6dc1761cf0a1a8a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/f2b56d6b73c34a9eb4f9ee23fb31134a] to archive 2024-11-15T22:17:18,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:17:18,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/3ab77223d5bf426a860effbe14ba018f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/3ab77223d5bf426a860effbe14ba018f 2024-11-15T22:17:18,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/53c631a4ac154d28b33ac92e397b4fbb to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/53c631a4ac154d28b33ac92e397b4fbb 2024-11-15T22:17:18,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/a4afe824a7bd41cb8be2f98b4aaccdef to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/a4afe824a7bd41cb8be2f98b4aaccdef 2024-11-15T22:17:18,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/6369fbc72af74c1896dd9cc0264e84d0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/6369fbc72af74c1896dd9cc0264e84d0 2024-11-15T22:17:18,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/7006aa1a138e498bb283ee62b9216137 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/7006aa1a138e498bb283ee62b9216137 2024-11-15T22:17:18,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/e843f4a864c245ceb36d330bec940524 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/e843f4a864c245ceb36d330bec940524 2024-11-15T22:17:18,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/148fdad423924675b7f7b575b8ee3da9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/148fdad423924675b7f7b575b8ee3da9 2024-11-15T22:17:18,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d51788963cda468bae4908fe8a4d9415 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d51788963cda468bae4908fe8a4d9415 2024-11-15T22:17:18,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/3fe637632a2945a8851fb933c759c888 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/3fe637632a2945a8851fb933c759c888 2024-11-15T22:17:18,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/7e9ce2a5107a41aba83e970e9d7c4e52 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/7e9ce2a5107a41aba83e970e9d7c4e52 2024-11-15T22:17:18,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/1cf9ae84262b4982955ab23cd039c8c8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/1cf9ae84262b4982955ab23cd039c8c8 2024-11-15T22:17:18,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/9d70cfdc6040447bb0536eb3f3deac0d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/9d70cfdc6040447bb0536eb3f3deac0d 2024-11-15T22:17:18,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/936d95074be6497cb39ec7bbd9dffc53 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/936d95074be6497cb39ec7bbd9dffc53 2024-11-15T22:17:18,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d566dae6d00040bd9b01189a6a909393 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d566dae6d00040bd9b01189a6a909393 2024-11-15T22:17:18,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/77a440bc49ec48ee98389fe764ed261a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/77a440bc49ec48ee98389fe764ed261a 2024-11-15T22:17:18,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/134fcbc7726c4251802d3a7acee8ec5f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/134fcbc7726c4251802d3a7acee8ec5f 2024-11-15T22:17:18,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/be60e0bec061414191b63b93b9affdf3 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/be60e0bec061414191b63b93b9affdf3 2024-11-15T22:17:18,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d5822703e40742ee9cfb686497049b6f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/d5822703e40742ee9cfb686497049b6f 2024-11-15T22:17:18,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/97942f56e8ef4ec78708611cf0a3f3cf to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/97942f56e8ef4ec78708611cf0a3f3cf 2024-11-15T22:17:18,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/77cf39c13b9241deb845af5886d5f8e1 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/77cf39c13b9241deb845af5886d5f8e1 2024-11-15T22:17:18,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/2964516225934b708a308699f0fe1dff to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/2964516225934b708a308699f0fe1dff 2024-11-15T22:17:18,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/725c1295e62e4325a6dc1761cf0a1a8a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/725c1295e62e4325a6dc1761cf0a1a8a 2024-11-15T22:17:18,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/f2b56d6b73c34a9eb4f9ee23fb31134a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/f2b56d6b73c34a9eb4f9ee23fb31134a 2024-11-15T22:17:18,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/bc8325c2f8eb4da0a770c8658caba681, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/7516347337984f3dbd703fdcd0c0cb57, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/8d37a8d72f504f4db75409dd639e86a6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/d0238f70861541e2a3661ab3fe683524, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/1b8e780adfd249a4b386866ca7d48102, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/90b45ce42dab4586ab92fe93de4bddea, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/12846108594b412bb124734ec671533a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/489db5cd39e642b5917789df0950e4c6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/55de7b2d0e534ad3b8f78717908ea077, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/d07a0a36756247999bfe28dfe80561d7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/f40500ac4dca476381347703def7d06c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/1c8fc7c313fb4f98a6f09ac56d3352b6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/497cb6512ff74dc3b36f985b8e8d2ed6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/8d4969b0467e4814b2acc0107e526fa8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/a248d6b85e584bb8954503234458e14d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/536517ab0ac64e6f8550f965f12bade3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/b401f4e10ff74006b579426deaa32c64, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/3614c1678dee4ad981a62bdc1e931f86, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/175cf533f5ae463b8e44662b29abd225, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/6e19f4590c704521b602bb7ac5590961, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/9f85e013c9c344cc943d90716b7aed61, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/2d89b3ad336c458bba40631b9a59bd29, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/e82b1fc81e9d4929bddca541813df5aa] to archive 2024-11-15T22:17:18,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:17:18,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/bc8325c2f8eb4da0a770c8658caba681 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/bc8325c2f8eb4da0a770c8658caba681 2024-11-15T22:17:18,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/7516347337984f3dbd703fdcd0c0cb57 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/7516347337984f3dbd703fdcd0c0cb57 2024-11-15T22:17:18,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/8d37a8d72f504f4db75409dd639e86a6 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/8d37a8d72f504f4db75409dd639e86a6 2024-11-15T22:17:18,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/d0238f70861541e2a3661ab3fe683524 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/d0238f70861541e2a3661ab3fe683524 2024-11-15T22:17:18,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/1b8e780adfd249a4b386866ca7d48102 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/1b8e780adfd249a4b386866ca7d48102 2024-11-15T22:17:18,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/90b45ce42dab4586ab92fe93de4bddea to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/90b45ce42dab4586ab92fe93de4bddea 2024-11-15T22:17:18,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/12846108594b412bb124734ec671533a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/12846108594b412bb124734ec671533a 2024-11-15T22:17:18,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/489db5cd39e642b5917789df0950e4c6 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/489db5cd39e642b5917789df0950e4c6 2024-11-15T22:17:18,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/55de7b2d0e534ad3b8f78717908ea077 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/55de7b2d0e534ad3b8f78717908ea077 2024-11-15T22:17:18,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/d07a0a36756247999bfe28dfe80561d7 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/d07a0a36756247999bfe28dfe80561d7 2024-11-15T22:17:18,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/f40500ac4dca476381347703def7d06c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/f40500ac4dca476381347703def7d06c 2024-11-15T22:17:18,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/1c8fc7c313fb4f98a6f09ac56d3352b6 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/1c8fc7c313fb4f98a6f09ac56d3352b6 2024-11-15T22:17:18,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/497cb6512ff74dc3b36f985b8e8d2ed6 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/497cb6512ff74dc3b36f985b8e8d2ed6 2024-11-15T22:17:18,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/8d4969b0467e4814b2acc0107e526fa8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/8d4969b0467e4814b2acc0107e526fa8 2024-11-15T22:17:18,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/a248d6b85e584bb8954503234458e14d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/a248d6b85e584bb8954503234458e14d 2024-11-15T22:17:18,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/536517ab0ac64e6f8550f965f12bade3 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/536517ab0ac64e6f8550f965f12bade3 2024-11-15T22:17:18,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/b401f4e10ff74006b579426deaa32c64 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/b401f4e10ff74006b579426deaa32c64 2024-11-15T22:17:18,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/3614c1678dee4ad981a62bdc1e931f86 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/3614c1678dee4ad981a62bdc1e931f86 2024-11-15T22:17:18,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/175cf533f5ae463b8e44662b29abd225 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/175cf533f5ae463b8e44662b29abd225 2024-11-15T22:17:18,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/6e19f4590c704521b602bb7ac5590961 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/6e19f4590c704521b602bb7ac5590961 2024-11-15T22:17:18,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/9f85e013c9c344cc943d90716b7aed61 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/9f85e013c9c344cc943d90716b7aed61 2024-11-15T22:17:18,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/2d89b3ad336c458bba40631b9a59bd29 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/2d89b3ad336c458bba40631b9a59bd29 2024-11-15T22:17:18,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/e82b1fc81e9d4929bddca541813df5aa to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/e82b1fc81e9d4929bddca541813df5aa 2024-11-15T22:17:18,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/da486add07b14a0e984a45f32e00833d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/69cc6fab5d7844c8a8a3f1c9c36da857, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/204ef1bb6ac24764984fbf21bcd84f6a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/d69b7fe5c6a34477b5ccc9c0fe24f9cc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/bdacf545453c4c0e86cc7a3ced1ab3d9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/46a7a177224b461eb4a107d090dce3c3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/b25174f22a064b9a8a1efe1cbe3edd8c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/60e30d3cd7ff4aa1968b39d76e416e2d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/9447968ce6084bab93c980a781d2f50f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/8f06802e78e94dbfbdbd31172f149a7c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/150a480a566b4f6f8381e97dae15dd19, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/8f00194263ca414682d9446cac5b3574, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/515e1d8cf5d94b4ebd67cf0d250255af, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/103766b7fa96423eb53998f9c1f5ca1f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/7128351b1ce9426d81b135ab07bbd971, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/13e4aadd3f7f41b5b6b85764d414321e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/983649c67a7b4640999f7ba84f92449a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/24ac16e02b0f4175920c93c6b017c76d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/bc5e7cf4f33c45d5931633b9d9ad6e05, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/67febe58ce69411491c27256c9f32c20, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/7dc8ecc601524b268159ea378bf52710, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/183774bcb4804ff59d3810fa29c890fe, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/a5ace5be68d84cf49f14b51d462d67b2] to archive 2024-11-15T22:17:18,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:17:18,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/da486add07b14a0e984a45f32e00833d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/da486add07b14a0e984a45f32e00833d 2024-11-15T22:17:18,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/69cc6fab5d7844c8a8a3f1c9c36da857 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/69cc6fab5d7844c8a8a3f1c9c36da857 2024-11-15T22:17:18,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/204ef1bb6ac24764984fbf21bcd84f6a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/204ef1bb6ac24764984fbf21bcd84f6a 2024-11-15T22:17:18,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/d69b7fe5c6a34477b5ccc9c0fe24f9cc to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/d69b7fe5c6a34477b5ccc9c0fe24f9cc 2024-11-15T22:17:18,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/bdacf545453c4c0e86cc7a3ced1ab3d9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/bdacf545453c4c0e86cc7a3ced1ab3d9 2024-11-15T22:17:18,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/46a7a177224b461eb4a107d090dce3c3 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/46a7a177224b461eb4a107d090dce3c3 2024-11-15T22:17:18,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/b25174f22a064b9a8a1efe1cbe3edd8c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/b25174f22a064b9a8a1efe1cbe3edd8c 2024-11-15T22:17:18,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/60e30d3cd7ff4aa1968b39d76e416e2d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/60e30d3cd7ff4aa1968b39d76e416e2d 2024-11-15T22:17:18,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/9447968ce6084bab93c980a781d2f50f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/9447968ce6084bab93c980a781d2f50f 2024-11-15T22:17:18,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/8f06802e78e94dbfbdbd31172f149a7c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/8f06802e78e94dbfbdbd31172f149a7c 2024-11-15T22:17:18,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/150a480a566b4f6f8381e97dae15dd19 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/150a480a566b4f6f8381e97dae15dd19 2024-11-15T22:17:18,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/8f00194263ca414682d9446cac5b3574 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/8f00194263ca414682d9446cac5b3574 2024-11-15T22:17:18,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/515e1d8cf5d94b4ebd67cf0d250255af to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/515e1d8cf5d94b4ebd67cf0d250255af 2024-11-15T22:17:18,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/103766b7fa96423eb53998f9c1f5ca1f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/103766b7fa96423eb53998f9c1f5ca1f 2024-11-15T22:17:18,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/7128351b1ce9426d81b135ab07bbd971 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/7128351b1ce9426d81b135ab07bbd971 2024-11-15T22:17:18,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/13e4aadd3f7f41b5b6b85764d414321e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/13e4aadd3f7f41b5b6b85764d414321e 2024-11-15T22:17:18,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/983649c67a7b4640999f7ba84f92449a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/983649c67a7b4640999f7ba84f92449a 2024-11-15T22:17:18,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/24ac16e02b0f4175920c93c6b017c76d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/24ac16e02b0f4175920c93c6b017c76d 2024-11-15T22:17:18,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/bc5e7cf4f33c45d5931633b9d9ad6e05 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/bc5e7cf4f33c45d5931633b9d9ad6e05 2024-11-15T22:17:18,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/67febe58ce69411491c27256c9f32c20 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/67febe58ce69411491c27256c9f32c20 2024-11-15T22:17:18,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/7dc8ecc601524b268159ea378bf52710 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/7dc8ecc601524b268159ea378bf52710 2024-11-15T22:17:18,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/183774bcb4804ff59d3810fa29c890fe to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/183774bcb4804ff59d3810fa29c890fe 2024-11-15T22:17:18,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/a5ace5be68d84cf49f14b51d462d67b2 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/a5ace5be68d84cf49f14b51d462d67b2 2024-11-15T22:17:18,324 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/recovered.edits/373.seqid, newMaxSeqId=373, maxSeqId=4 2024-11-15T22:17:18,325 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. 2024-11-15T22:17:18,325 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1676): Region close journal for 94761791c1172512c89bb88a26d6fda0: Waiting for close lock at 1731709036106Running coprocessor pre-close hooks at 1731709036106Disabling compacts and flushes for region at 1731709036106Disabling writes for close at 1731709036845 (+739 ms)Obtaining lock to block concurrent updates at 1731709036845Preparing flush snapshotting stores in 94761791c1172512c89bb88a26d6fda0 at 1731709036845Finished memstore snapshotting TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731709036846 (+1 ms)Flushing stores of TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0. at 1731709036851 (+5 ms)Flushing 94761791c1172512c89bb88a26d6fda0/A: creating writer at 1731709036851Flushing mob file 94761791c1172512c89bb88a26d6fda0/A: appending metadata at 1731709036872 (+21 ms)Flushing mob file 94761791c1172512c89bb88a26d6fda0/A: closing flushed file at 1731709036872Flushing 94761791c1172512c89bb88a26d6fda0/A: appending metadata at 1731709036933 (+61 ms)Flushing 94761791c1172512c89bb88a26d6fda0/A: closing flushed file at 1731709036933Flushing 94761791c1172512c89bb88a26d6fda0/B: creating writer at 1731709037351 (+418 ms)Flushing 94761791c1172512c89bb88a26d6fda0/B: appending metadata at 1731709037354 (+3 ms)Flushing 94761791c1172512c89bb88a26d6fda0/B: closing flushed file at 1731709037354Flushing 94761791c1172512c89bb88a26d6fda0/C: creating writer at 1731709037768 (+414 ms)Flushing 94761791c1172512c89bb88a26d6fda0/C: appending metadata at 1731709037771 (+3 ms)Flushing 94761791c1172512c89bb88a26d6fda0/C: closing flushed file at 1731709037771Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@685f2fd9: reopening flushed file at 1731709038194 (+423 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ded4ac: reopening flushed file at 1731709038200 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66b28c5d: reopening flushed file at 1731709038207 (+7 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 94761791c1172512c89bb88a26d6fda0 in 1369ms, sequenceid=370, compaction requested=true at 1731709038215 (+8 ms)Writing region close event to WAL at 1731709038321 (+106 ms)Running coprocessor post-close hooks at 1731709038324 (+3 ms)Closed at 1731709038325 (+1 ms) 2024-11-15T22:17:18,326 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(157): Closed 94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:18,327 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=94761791c1172512c89bb88a26d6fda0, regionState=CLOSED 2024-11-15T22:17:18,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:17:18,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-15T22:17:18,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseRegionProcedure 94761791c1172512c89bb88a26d6fda0, server=e611192d6313,38779,1731708927034 in 2.3750 sec 2024-11-15T22:17:18,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-15T22:17:18,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94761791c1172512c89bb88a26d6fda0, UNASSIGN in 2.3790 sec 2024-11-15T22:17:18,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-15T22:17:18,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3830 sec 2024-11-15T22:17:18,334 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731709038334"}]},"ts":"1731709038334"} 2024-11-15T22:17:18,336 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T22:17:18,336 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T22:17:18,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3930 sec 2024-11-15T22:17:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T22:17:20,108 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:20,114 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T22:17:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:20,118 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T22:17:20,124 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=107, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:20,160 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,200 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/recovered.edits] 2024-11-15T22:17:20,231 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/84892d9c643c487f8e787f318eb3901a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/84892d9c643c487f8e787f318eb3901a 2024-11-15T22:17:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T22:17:20,245 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/e7a697882e964b758a2ec3570417ed31 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/e7a697882e964b758a2ec3570417ed31 2024-11-15T22:17:20,249 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/ed87e9fdc81a4af19b0fcd531c7f624e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/ed87e9fdc81a4af19b0fcd531c7f624e 2024-11-15T22:17:20,253 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/f0ed630f431248aaa8cf8f6bd06c6bc8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/A/f0ed630f431248aaa8cf8f6bd06c6bc8 2024-11-15T22:17:20,265 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/0eea00bdf0114f0f892d1280ab8eb11e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/0eea00bdf0114f0f892d1280ab8eb11e 2024-11-15T22:17:20,271 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/150589cff6514bb3844385de56181174 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/150589cff6514bb3844385de56181174 2024-11-15T22:17:20,274 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/19b52fbf8f0943db80f505b60e965845 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/19b52fbf8f0943db80f505b60e965845 2024-11-15T22:17:20,275 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/eb7a903103bf49e0b381e62ac9fe4446 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/B/eb7a903103bf49e0b381e62ac9fe4446 2024-11-15T22:17:20,300 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/377dac47874f442293818c04e29c375a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/377dac47874f442293818c04e29c375a 2024-11-15T22:17:20,316 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/9936fea1159c4dd5a8ce222ae02a3130 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/9936fea1159c4dd5a8ce222ae02a3130 2024-11-15T22:17:20,331 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/c64cff8dbfb24779b25ca22b75a346d1 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/c64cff8dbfb24779b25ca22b75a346d1 2024-11-15T22:17:20,340 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/f63c308d07ea466aa1027662435e5824 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/C/f63c308d07ea466aa1027662435e5824 2024-11-15T22:17:20,356 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/recovered.edits/373.seqid to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0/recovered.edits/373.seqid 2024-11-15T22:17:20,361 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,362 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T22:17:20,363 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T22:17:20,370 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T22:17:20,412 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151a1da8d19594470bbd3dad392c5a237d_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151a1da8d19594470bbd3dad392c5a237d_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,436 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111541343d2deacf4aee9f7af8ee9bee59ef_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111541343d2deacf4aee9f7af8ee9bee59ef_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,445 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154af17d9388a44c05bccd2d1f5fdb37b2_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154af17d9388a44c05bccd2d1f5fdb37b2_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T22:17:20,453 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154fc16b9322b44d058d022bf7ce6a07f0_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154fc16b9322b44d058d022bf7ce6a07f0_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,457 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155f8020ba07094f19afb5b73b419fcf73_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155f8020ba07094f19afb5b73b419fcf73_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,467 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156e13751257b04ebfac75d41752d97ad9_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156e13751257b04ebfac75d41752d97ad9_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,471 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158165fe5a6c8e435099e15cc379a74bd2_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158165fe5a6c8e435099e15cc379a74bd2_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,473 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111581fdbf8d0a904a09bc983715ce6d0409_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111581fdbf8d0a904a09bc983715ce6d0409_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,484 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115847ce931ad624d948e331bf012e9574f_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115847ce931ad624d948e331bf012e9574f_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,487 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111584fb2000294741409004278e31e7b242_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111584fb2000294741409004278e31e7b242_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,490 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159945f141bf2f4227b948367e406a87bf_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159945f141bf2f4227b948367e406a87bf_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,502 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159e9360a5cf544a97a8b20ce88e0cb631_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159e9360a5cf544a97a8b20ce88e0cb631_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,521 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159f2b427a0f7347dfa5014b4c8033f099_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159f2b427a0f7347dfa5014b4c8033f099_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,524 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115aa583939d163495b96043c658a20e727_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115aa583939d163495b96043c658a20e727_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,528 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b4350214ec1e4072b74b860c34a07521_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b4350214ec1e4072b74b860c34a07521_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,532 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b5c4b00c9e094484be7d81d8b08c7cb7_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b5c4b00c9e094484be7d81d8b08c7cb7_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,538 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c2d6c73aec7e4ae4887c3cd5c9a28d64_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c2d6c73aec7e4ae4887c3cd5c9a28d64_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,539 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d6fa0c245c5a4e0baaf08227bded4a27_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d6fa0c245c5a4e0baaf08227bded4a27_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,540 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115df63a895f19c4280b52ad2397985a392_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115df63a895f19c4280b52ad2397985a392_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,542 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115eb4060d8ce6a4d52b23f7c1275da3829_94761791c1172512c89bb88a26d6fda0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115eb4060d8ce6a4d52b23f7c1275da3829_94761791c1172512c89bb88a26d6fda0 2024-11-15T22:17:20,542 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T22:17:20,545 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=107, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:20,547 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T22:17:20,553 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T22:17:20,555 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=107, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:20,555 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T22:17:20,555 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731709040555"}]},"ts":"9223372036854775807"} 2024-11-15T22:17:20,558 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T22:17:20,558 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 94761791c1172512c89bb88a26d6fda0, NAME => 'TestAcidGuarantees,,1731709013197.94761791c1172512c89bb88a26d6fda0.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T22:17:20,558 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T22:17:20,558 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731709040558"}]},"ts":"9223372036854775807"} 2024-11-15T22:17:20,563 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T22:17:20,563 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=107, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:20,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 450 msec 2024-11-15T22:17:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T22:17:20,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T22:17:20,763 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:20,781 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=455 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1187 (was 1297), ProcessCount=11 (was 11), AvailableMemoryMB=251 (was 256) 2024-11-15T22:17:20,792 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=1187, ProcessCount=11, AvailableMemoryMB=251 2024-11-15T22:17:20,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:17:20,794 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:17:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:20,797 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T22:17:20,797 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:20,797 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 108 2024-11-15T22:17:20,798 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T22:17:20,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-15T22:17:20,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742258_1434 (size=1061) 2024-11-15T22:17:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-15T22:17:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-15T22:17:21,215 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e 2024-11-15T22:17:21,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742259_1435 (size=53) 2024-11-15T22:17:21,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:17:21,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 00af3e32e9b483f8e15f8f4708045315, disabling compactions & flushes 2024-11-15T22:17:21,244 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:21,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:21,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. after waiting 0 ms 2024-11-15T22:17:21,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:21,244 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:21,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 00af3e32e9b483f8e15f8f4708045315: Waiting for close lock at 1731709041244Disabling compacts and flushes for region at 1731709041244Disabling writes for close at 1731709041244Writing region close event to WAL at 1731709041244Closed at 1731709041244 2024-11-15T22:17:21,245 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T22:17:21,246 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731709041245"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731709041245"}]},"ts":"1731709041245"} 2024-11-15T22:17:21,247 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T22:17:21,248 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T22:17:21,248 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731709041248"}]},"ts":"1731709041248"} 2024-11-15T22:17:21,250 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T22:17:21,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00af3e32e9b483f8e15f8f4708045315, ASSIGN}] 2024-11-15T22:17:21,252 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00af3e32e9b483f8e15f8f4708045315, ASSIGN 2024-11-15T22:17:21,252 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00af3e32e9b483f8e15f8f4708045315, ASSIGN; state=OFFLINE, location=e611192d6313,38779,1731708927034; forceNewPlan=false, retain=false 2024-11-15T22:17:21,403 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=00af3e32e9b483f8e15f8f4708045315, regionState=OPENING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:17:21,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00af3e32e9b483f8e15f8f4708045315, ASSIGN because future has completed 2024-11-15T22:17:21,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:17:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-15T22:17:21,574 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:21,574 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7752): Opening region: {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:17:21,575 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:21,575 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:17:21,575 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7794): checking encryption for 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:21,575 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7797): checking classloading for 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:21,576 INFO [StoreOpener-00af3e32e9b483f8e15f8f4708045315-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:21,581 INFO [StoreOpener-00af3e32e9b483f8e15f8f4708045315-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:17:21,582 INFO [StoreOpener-00af3e32e9b483f8e15f8f4708045315-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00af3e32e9b483f8e15f8f4708045315 columnFamilyName A 2024-11-15T22:17:21,582 DEBUG [StoreOpener-00af3e32e9b483f8e15f8f4708045315-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:21,583 INFO [StoreOpener-00af3e32e9b483f8e15f8f4708045315-1 {}] regionserver.HStore(327): Store=00af3e32e9b483f8e15f8f4708045315/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:17:21,583 INFO [StoreOpener-00af3e32e9b483f8e15f8f4708045315-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:21,585 INFO [StoreOpener-00af3e32e9b483f8e15f8f4708045315-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:17:21,586 INFO [StoreOpener-00af3e32e9b483f8e15f8f4708045315-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00af3e32e9b483f8e15f8f4708045315 columnFamilyName B 2024-11-15T22:17:21,586 DEBUG [StoreOpener-00af3e32e9b483f8e15f8f4708045315-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:21,589 INFO [StoreOpener-00af3e32e9b483f8e15f8f4708045315-1 {}] regionserver.HStore(327): Store=00af3e32e9b483f8e15f8f4708045315/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:17:21,589 INFO [StoreOpener-00af3e32e9b483f8e15f8f4708045315-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:21,591 INFO [StoreOpener-00af3e32e9b483f8e15f8f4708045315-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:17:21,592 INFO [StoreOpener-00af3e32e9b483f8e15f8f4708045315-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00af3e32e9b483f8e15f8f4708045315 columnFamilyName C 2024-11-15T22:17:21,592 DEBUG [StoreOpener-00af3e32e9b483f8e15f8f4708045315-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:21,596 INFO [StoreOpener-00af3e32e9b483f8e15f8f4708045315-1 {}] regionserver.HStore(327): Store=00af3e32e9b483f8e15f8f4708045315/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:17:21,597 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:21,597 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1038): replaying wal for 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:21,600 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:21,601 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:21,602 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1048): stopping wal replay for 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:21,602 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1060): Cleaning up temporary data for 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:21,602 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:17:21,603 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1093): writing seq id for 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:21,611 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:17:21,615 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1114): Opened 00af3e32e9b483f8e15f8f4708045315; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73961998, jitterRate=0.10211965441703796}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:17:21,615 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:21,615 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1006): Region open journal for 00af3e32e9b483f8e15f8f4708045315: Running coprocessor pre-open hook at 1731709041575Writing region info on filesystem at 1731709041575Initializing all the Stores at 1731709041576 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731709041576Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731709041576Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731709041576Cleaning up temporary data from old regions at 1731709041602 (+26 ms)Running coprocessor post-open hooks at 1731709041615 (+13 ms)Region opened successfully at 1731709041615 2024-11-15T22:17:21,618 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., pid=110, masterSystemTime=1731709041564 2024-11-15T22:17:21,626 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:21,626 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:21,631 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=00af3e32e9b483f8e15f8f4708045315, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:17:21,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:17:21,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-15T22:17:21,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; OpenRegionProcedure 00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 in 228 msec 2024-11-15T22:17:21,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-15T22:17:21,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00af3e32e9b483f8e15f8f4708045315, ASSIGN in 389 msec 2024-11-15T22:17:21,643 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T22:17:21,643 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731709041643"}]},"ts":"1731709041643"} 2024-11-15T22:17:21,653 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T22:17:21,654 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T22:17:21,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 860 msec 2024-11-15T22:17:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-15T22:17:21,932 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:21,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd12bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:21,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:21,934 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f878473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:21,936 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@786e09e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,937 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:21,940 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:21,941 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T22:17:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is e611192d6313,34969,1731708925722 2024-11-15T22:17:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1254797 2024-11-15T22:17:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T22:17:21,943 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T22:17:21,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@136bd6b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,945 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:21,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:21,946 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:21,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:21,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:21,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29bf5de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:21,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:21,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:21,948 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fdedb05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3128f70a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,952 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:21,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:21,954 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@182b9a4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:21,956 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:21,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e9ae24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4499c466, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:21,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:21,966 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e221945, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:21,968 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:21,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32d4f808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340f50c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:21,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:21,973 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be77d08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:21,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:21,974 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764e06d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ff7534, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:21,981 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:21,981 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d2740e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:21,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:21,984 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:21,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@928cb18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64cc712e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:22,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:22,007 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6758b8ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:22,009 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:22,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4f9325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6196a59c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:22,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:22,013 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@224de7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:22,015 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6463b671, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fcc4da7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,017 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:22,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:22,019 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14277c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:22,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:22,021 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da7779d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77291cb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:22,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:22,023 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:22,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:22,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:22,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c03b790, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:22,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:22,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:22,030 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e701331, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@541036fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,037 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:22,037 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:22,038 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e3c852, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:22,039 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c942f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:22,042 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:17:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-15T22:17:22,044 DEBUG [Thread-1955 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:22,044 DEBUG [Thread-1972 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:22,045 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:17:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T22:17:22,046 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:17:22,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:17:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:22,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:22,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:22,050 DEBUG [Thread-1963 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:22,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:22,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:17:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:17:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:22,055 DEBUG [Thread-1961 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:22,056 DEBUG [Thread-1968 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:22,057 DEBUG [Thread-1968 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:22,058 DEBUG [Thread-1968 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:22,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:22,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:22,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:17:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:17:22,068 DEBUG [Thread-1959 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:22,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:17:22,071 DEBUG [Thread-1974 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:22,072 DEBUG [Thread-1957 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:22,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T22:17:22,073 DEBUG [Thread-1966 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:22,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:22,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:22,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:22,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:22,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:22,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:22,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:17:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:22,076 DEBUG [Thread-1970 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:22,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:22,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:22,077 DEBUG [Thread-1957 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:22,077 DEBUG [Thread-1957 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:22,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:17:22,085 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:22,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:17:22,085 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:22,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:17:22,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2] 2024-11-15T22:17:22,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/f10458dd6a7d4929aaf55e6906a5e318 is 50, key is test_row_0/A:col10/1731709042069/Put/seqid=0 2024-11-15T22:17:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T22:17:22,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1731709042118, max timestamp=1731709042173Num uniques 859; 2024-11-15T22:17:22,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1731709042118, max timestamp=1731709042173Num uniques 859; 2024-11-15T22:17:22,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T22:17:22,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T22:17:22,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1731709042118, max timestamp=1731709042173Num uniques 859; 2024-11-15T22:17:22,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T22:17:22,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742260_1436 (size=12001) 2024-11-15T22:17:22,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T22:17:22,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:22,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:22,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:22,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:22,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:22,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709102241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709102249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709102261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709102261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709102263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T22:17:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T22:17:22,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:22,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:22,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:22,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:22,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:22,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709102371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709102371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709102402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709102404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709102404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T22:17:22,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:22,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:22,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:22,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:22,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:22,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/f10458dd6a7d4929aaf55e6906a5e318 2024-11-15T22:17:22,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709102592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709102592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709102622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709102624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/34b9f0724dd748a08ec524f834b23eb6 is 50, key is test_row_0/B:col10/1731709042069/Put/seqid=0 2024-11-15T22:17:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709102633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T22:17:22,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742261_1437 (size=12001) 2024-11-15T22:17:22,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/34b9f0724dd748a08ec524f834b23eb6 2024-11-15T22:17:22,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T22:17:22,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:22,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:22,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:22,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:22,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:22,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/40dc6ac3d8664ac69ed4eed4abbbbc18 is 50, key is test_row_0/C:col10/1731709042069/Put/seqid=0 2024-11-15T22:17:22,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742262_1438 (size=12001) 2024-11-15T22:17:22,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/40dc6ac3d8664ac69ed4eed4abbbbc18 2024-11-15T22:17:22,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/f10458dd6a7d4929aaf55e6906a5e318 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f10458dd6a7d4929aaf55e6906a5e318 2024-11-15T22:17:22,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f10458dd6a7d4929aaf55e6906a5e318, entries=150, sequenceid=11, filesize=11.7 K 2024-11-15T22:17:22,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/34b9f0724dd748a08ec524f834b23eb6 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/34b9f0724dd748a08ec524f834b23eb6 2024-11-15T22:17:22,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/34b9f0724dd748a08ec524f834b23eb6, entries=150, sequenceid=11, filesize=11.7 K 2024-11-15T22:17:22,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/40dc6ac3d8664ac69ed4eed4abbbbc18 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/40dc6ac3d8664ac69ed4eed4abbbbc18 2024-11-15T22:17:22,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/40dc6ac3d8664ac69ed4eed4abbbbc18, entries=150, sequenceid=11, filesize=11.7 K 2024-11-15T22:17:22,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 00af3e32e9b483f8e15f8f4708045315 in 801ms, sequenceid=11, compaction requested=false 2024-11-15T22:17:22,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:22,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T22:17:22,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:22,893 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-15T22:17:22,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:22,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:22,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:22,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:22,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:22,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:22,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/3bd39ed63353417080b5d99aae909cac is 50, key is test_row_0/A:col10/1731709042259/Put/seqid=0 2024-11-15T22:17:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:22,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:22,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742263_1439 (size=12001) 2024-11-15T22:17:22,965 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/3bd39ed63353417080b5d99aae909cac 2024-11-15T22:17:22,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/d4262c8777504c8aa3027b3314f7cbdf is 50, key is test_row_0/B:col10/1731709042259/Put/seqid=0 2024-11-15T22:17:22,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709102961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:22,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709102963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:22,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709102964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709102962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709102967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742264_1440 (size=12001) 2024-11-15T22:17:23,033 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/d4262c8777504c8aa3027b3314f7cbdf 2024-11-15T22:17:23,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/bb6772f854454553bedf147fb4416dcc is 50, key is test_row_0/C:col10/1731709042259/Put/seqid=0 2024-11-15T22:17:23,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709103102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742265_1441 (size=12001) 2024-11-15T22:17:23,132 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/bb6772f854454553bedf147fb4416dcc 2024-11-15T22:17:23,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709103111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709103111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709103111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/3bd39ed63353417080b5d99aae909cac as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/3bd39ed63353417080b5d99aae909cac 2024-11-15T22:17:23,171 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/3bd39ed63353417080b5d99aae909cac, entries=150, sequenceid=46, filesize=11.7 K 2024-11-15T22:17:23,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/d4262c8777504c8aa3027b3314f7cbdf as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/d4262c8777504c8aa3027b3314f7cbdf 2024-11-15T22:17:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T22:17:23,208 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/d4262c8777504c8aa3027b3314f7cbdf, entries=150, sequenceid=46, filesize=11.7 K 2024-11-15T22:17:23,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/bb6772f854454553bedf147fb4416dcc as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/bb6772f854454553bedf147fb4416dcc 2024-11-15T22:17:23,265 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/bb6772f854454553bedf147fb4416dcc, entries=150, sequenceid=46, filesize=11.7 K 2024-11-15T22:17:23,267 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for 00af3e32e9b483f8e15f8f4708045315 in 374ms, sequenceid=46, compaction requested=false 2024-11-15T22:17:23,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:23,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:23,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-15T22:17:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-15T22:17:23,286 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-15T22:17:23,286 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2300 sec 2024-11-15T22:17:23,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.2450 sec 2024-11-15T22:17:23,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T22:17:23,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:23,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:23,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:23,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:23,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:23,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:23,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/1942fc4e9c164a06af9ed8686ea1bd13 is 50, key is test_row_0/A:col10/1731709042960/Put/seqid=0 2024-11-15T22:17:23,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742266_1442 (size=12001) 2024-11-15T22:17:23,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/1942fc4e9c164a06af9ed8686ea1bd13 2024-11-15T22:17:23,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/c7c21bc3078740ddbd61bfda3a9be77a is 50, key is test_row_0/B:col10/1731709042960/Put/seqid=0 2024-11-15T22:17:23,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731709043409, max timestamp=1731709043448Num uniques 859; 2024-11-15T22:17:23,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T22:17:23,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731709043409, max timestamp=1731709043448Num uniques 859; 2024-11-15T22:17:23,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T22:17:23,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731709043409, max timestamp=1731709043448Num uniques 859; 2024-11-15T22:17:23,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T22:17:23,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742267_1443 (size=12001) 2024-11-15T22:17:23,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709103537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709103555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709103556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709103559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709103573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709103688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709103693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709103693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709103693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709103702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,869 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T22:17:23,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/c7c21bc3078740ddbd61bfda3a9be77a 2024-11-15T22:17:23,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/7519b20c311643f4b7337073a22c5be3 is 50, key is test_row_0/C:col10/1731709042960/Put/seqid=0 2024-11-15T22:17:23,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709103915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709103915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709103917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709103917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709103924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:23,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742268_1444 (size=12001) 2024-11-15T22:17:23,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/7519b20c311643f4b7337073a22c5be3 2024-11-15T22:17:24,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/1942fc4e9c164a06af9ed8686ea1bd13 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/1942fc4e9c164a06af9ed8686ea1bd13 2024-11-15T22:17:24,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/1942fc4e9c164a06af9ed8686ea1bd13, entries=150, sequenceid=56, filesize=11.7 K 2024-11-15T22:17:24,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/c7c21bc3078740ddbd61bfda3a9be77a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c7c21bc3078740ddbd61bfda3a9be77a 2024-11-15T22:17:24,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c7c21bc3078740ddbd61bfda3a9be77a, entries=150, sequenceid=56, filesize=11.7 K 2024-11-15T22:17:24,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/7519b20c311643f4b7337073a22c5be3 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/7519b20c311643f4b7337073a22c5be3 2024-11-15T22:17:24,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/7519b20c311643f4b7337073a22c5be3, entries=150, sequenceid=56, filesize=11.7 K 2024-11-15T22:17:24,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 00af3e32e9b483f8e15f8f4708045315 in 799ms, sequenceid=56, compaction requested=true 2024-11-15T22:17:24,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:24,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:24,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:24,128 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:24,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:24,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:24,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:24,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:17:24,128 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:24,140 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:24,140 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/A is initiating minor compaction (all files) 2024-11-15T22:17:24,140 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/A in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:24,140 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f10458dd6a7d4929aaf55e6906a5e318, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/3bd39ed63353417080b5d99aae909cac, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/1942fc4e9c164a06af9ed8686ea1bd13] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=35.2 K 2024-11-15T22:17:24,144 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:24,144 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/B is initiating minor compaction (all files) 2024-11-15T22:17:24,144 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/B in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:24,144 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/34b9f0724dd748a08ec524f834b23eb6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/d4262c8777504c8aa3027b3314f7cbdf, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c7c21bc3078740ddbd61bfda3a9be77a] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=35.2 K 2024-11-15T22:17:24,145 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting f10458dd6a7d4929aaf55e6906a5e318, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731709042069 2024-11-15T22:17:24,148 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bd39ed63353417080b5d99aae909cac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731709042227 2024-11-15T22:17:24,148 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 34b9f0724dd748a08ec524f834b23eb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731709042069 2024-11-15T22:17:24,151 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1942fc4e9c164a06af9ed8686ea1bd13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731709042927 2024-11-15T22:17:24,155 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d4262c8777504c8aa3027b3314f7cbdf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731709042227 2024-11-15T22:17:24,160 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c7c21bc3078740ddbd61bfda3a9be77a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731709042927 2024-11-15T22:17:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T22:17:24,201 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:24,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:17:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-15T22:17:24,213 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:17:24,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T22:17:24,214 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:17:24,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:17:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:24,238 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#A#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:24,239 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/41d49b54f8c8451388365107dd93fa9a is 50, key is test_row_0/A:col10/1731709042960/Put/seqid=0 2024-11-15T22:17:24,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-15T22:17:24,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:24,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:24,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:24,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:24,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:24,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:24,245 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#B#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:24,245 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/36385214389c408abf4b78d422fe0172 is 50, key is test_row_0/B:col10/1731709042960/Put/seqid=0 2024-11-15T22:17:24,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/e800fef70de14940bd0f50035d7263a9 is 50, key is test_row_0/A:col10/1731709044235/Put/seqid=0 2024-11-15T22:17:24,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709104267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709104268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709104271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742269_1445 (size=12104) 2024-11-15T22:17:24,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709104277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T22:17:24,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709104294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742270_1446 (size=12104) 2024-11-15T22:17:24,346 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/41d49b54f8c8451388365107dd93fa9a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/41d49b54f8c8451388365107dd93fa9a 2024-11-15T22:17:24,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T22:17:24,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:24,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:24,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:24,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:24,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:24,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742271_1447 (size=14341) 2024-11-15T22:17:24,384 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/A of 00af3e32e9b483f8e15f8f4708045315 into 41d49b54f8c8451388365107dd93fa9a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:24,384 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:24,384 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/A, priority=13, startTime=1731709044128; duration=0sec 2024-11-15T22:17:24,384 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:24,385 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:A 2024-11-15T22:17:24,385 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:24,392 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:24,392 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/C is initiating minor compaction (all files) 2024-11-15T22:17:24,392 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/C in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:24,392 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/40dc6ac3d8664ac69ed4eed4abbbbc18, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/bb6772f854454553bedf147fb4416dcc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/7519b20c311643f4b7337073a22c5be3] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=35.2 K 2024-11-15T22:17:24,393 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40dc6ac3d8664ac69ed4eed4abbbbc18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731709042069 2024-11-15T22:17:24,396 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb6772f854454553bedf147fb4416dcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731709042227 2024-11-15T22:17:24,400 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7519b20c311643f4b7337073a22c5be3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731709042927 2024-11-15T22:17:24,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709104404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709104405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709104406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709104422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,430 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#C#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:24,431 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/024cbcb5324840a7bfa90e07370de7e4 is 50, key is test_row_0/C:col10/1731709042960/Put/seqid=0 2024-11-15T22:17:24,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709104432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742272_1448 (size=12104) 2024-11-15T22:17:24,494 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/024cbcb5324840a7bfa90e07370de7e4 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/024cbcb5324840a7bfa90e07370de7e4 2024-11-15T22:17:24,502 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/C of 00af3e32e9b483f8e15f8f4708045315 into 024cbcb5324840a7bfa90e07370de7e4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:24,503 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:24,503 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/C, priority=13, startTime=1731709044128; duration=0sec 2024-11-15T22:17:24,503 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:24,503 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:C 2024-11-15T22:17:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T22:17:24,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T22:17:24,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:24,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:24,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:24,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:24,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:24,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709104624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709104629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709104632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709104634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,651 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T22:17:24,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709104668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T22:17:24,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:24,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:24,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:24,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:24,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:24,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/e800fef70de14940bd0f50035d7263a9 2024-11-15T22:17:24,801 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/36385214389c408abf4b78d422fe0172 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/36385214389c408abf4b78d422fe0172 2024-11-15T22:17:24,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/5adb63cf185e4146b61a5b30822872d5 is 50, key is test_row_0/B:col10/1731709044235/Put/seqid=0 2024-11-15T22:17:24,817 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/B of 00af3e32e9b483f8e15f8f4708045315 into 36385214389c408abf4b78d422fe0172(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:24,817 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:24,817 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/B, priority=13, startTime=1731709044128; duration=0sec 2024-11-15T22:17:24,817 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:24,817 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:B 2024-11-15T22:17:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T22:17:24,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T22:17:24,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:24,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:24,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:24,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:24,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:24,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742273_1449 (size=12001) 2024-11-15T22:17:24,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709104941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709104952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709104952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709104962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:24,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709104982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:25,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T22:17:25,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:25,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:25,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:25,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T22:17:25,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:25,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:25,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:25,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/5adb63cf185e4146b61a5b30822872d5 2024-11-15T22:17:25,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/38be5937d6464411ac1ab3f999d19d7e is 50, key is test_row_0/C:col10/1731709044235/Put/seqid=0 2024-11-15T22:17:25,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T22:17:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:25,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T22:17:25,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742274_1450 (size=12001) 2024-11-15T22:17:25,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709105462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:25,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709105462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:25,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709105462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:25,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:25,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:25,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:25,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709105491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:25,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:25,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T22:17:25,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:25,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:25,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:25,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709105512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:25,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:25,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:25,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:25,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T22:17:25,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:25,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:25,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:25,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/38be5937d6464411ac1ab3f999d19d7e 2024-11-15T22:17:25,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/e800fef70de14940bd0f50035d7263a9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/e800fef70de14940bd0f50035d7263a9 2024-11-15T22:17:25,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T22:17:25,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:25,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:25,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:25,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:25,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/e800fef70de14940bd0f50035d7263a9, entries=200, sequenceid=92, filesize=14.0 K 2024-11-15T22:17:25,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/5adb63cf185e4146b61a5b30822872d5 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5adb63cf185e4146b61a5b30822872d5 2024-11-15T22:17:25,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5adb63cf185e4146b61a5b30822872d5, entries=150, sequenceid=92, filesize=11.7 K 2024-11-15T22:17:25,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/38be5937d6464411ac1ab3f999d19d7e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/38be5937d6464411ac1ab3f999d19d7e 2024-11-15T22:17:25,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/38be5937d6464411ac1ab3f999d19d7e, entries=150, sequenceid=92, filesize=11.7 K 2024-11-15T22:17:25,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 00af3e32e9b483f8e15f8f4708045315 in 1627ms, sequenceid=92, compaction requested=false 2024-11-15T22:17:25,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:25,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T22:17:25,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:25,977 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-15T22:17:25,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:25,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:25,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:25,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:25,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:25,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:25,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/eb874ee30cd245808299e6e3fe3d4c30 is 50, key is test_row_2/A:col10/1731709044286/Put/seqid=0 2024-11-15T22:17:26,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742275_1451 (size=7315) 2024-11-15T22:17:26,014 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/eb874ee30cd245808299e6e3fe3d4c30 2024-11-15T22:17:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/c7da98e1a98447078b760730459075ed is 50, key is test_row_2/B:col10/1731709044286/Put/seqid=0 2024-11-15T22:17:26,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742276_1452 (size=7315) 2024-11-15T22:17:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T22:17:26,459 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/c7da98e1a98447078b760730459075ed 2024-11-15T22:17:26,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/db376a74f70743f9b614d59bfd9da0f5 is 50, key is test_row_2/C:col10/1731709044286/Put/seqid=0 2024-11-15T22:17:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:26,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:26,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742277_1453 (size=7315) 2024-11-15T22:17:26,511 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/db376a74f70743f9b614d59bfd9da0f5 2024-11-15T22:17:26,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/eb874ee30cd245808299e6e3fe3d4c30 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/eb874ee30cd245808299e6e3fe3d4c30 2024-11-15T22:17:26,521 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/eb874ee30cd245808299e6e3fe3d4c30, entries=50, sequenceid=104, filesize=7.1 K 2024-11-15T22:17:26,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/c7da98e1a98447078b760730459075ed as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c7da98e1a98447078b760730459075ed 2024-11-15T22:17:26,526 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c7da98e1a98447078b760730459075ed, entries=50, sequenceid=104, filesize=7.1 K 2024-11-15T22:17:26,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/db376a74f70743f9b614d59bfd9da0f5 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/db376a74f70743f9b614d59bfd9da0f5 2024-11-15T22:17:26,536 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/db376a74f70743f9b614d59bfd9da0f5, entries=50, sequenceid=104, filesize=7.1 K 2024-11-15T22:17:26,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731709046518, max timestamp=1731709046534Num uniques 859; 2024-11-15T22:17:26,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T22:17:26,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731709046518, max timestamp=1731709046534Num uniques 859; 2024-11-15T22:17:26,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T22:17:26,538 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=60.38 KB/61830 for 00af3e32e9b483f8e15f8f4708045315 in 561ms, sequenceid=104, compaction requested=true 2024-11-15T22:17:26,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:26,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:26,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-15T22:17:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-15T22:17:26,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731709046523, max timestamp=1731709046534Num uniques 859; 2024-11-15T22:17:26,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:17:26,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=42.49 KB heapSize=106.47 KB 2024-11-15T22:17:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:26,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/fa1c6a5b23954209bef61734f33c88dc is 50, key is test_row_0/A:col10/1731709046539/Put/seqid=0 2024-11-15T22:17:26,550 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-15T22:17:26,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3300 sec 2024-11-15T22:17:26,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.3460 sec 2024-11-15T22:17:26,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742278_1454 (size=19021) 2024-11-15T22:17:26,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/fa1c6a5b23954209bef61734f33c88dc 2024-11-15T22:17:26,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/8d69895111504ee0a7e9d49b6ff97d53 is 50, key is test_row_0/B:col10/1731709046539/Put/seqid=0 2024-11-15T22:17:26,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742279_1455 (size=12001) 2024-11-15T22:17:26,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/8d69895111504ee0a7e9d49b6ff97d53 2024-11-15T22:17:26,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/6fbbe4423113448aa292ccdec51ef380 is 50, key is test_row_0/C:col10/1731709046539/Put/seqid=0 2024-11-15T22:17:26,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731709046572, max timestamp=1731709046601Num uniques 859; 2024-11-15T22:17:26,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T22:17:26,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731709046572, max timestamp=1731709046601Num uniques 859; 2024-11-15T22:17:26,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T22:17:26,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742280_1456 (size=12001) 2024-11-15T22:17:26,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731709046572, max timestamp=1731709046601Num uniques 859; 2024-11-15T22:17:26,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T22:17:26,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/6fbbe4423113448aa292ccdec51ef380 2024-11-15T22:17:26,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/fa1c6a5b23954209bef61734f33c88dc as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/fa1c6a5b23954209bef61734f33c88dc 2024-11-15T22:17:26,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/fa1c6a5b23954209bef61734f33c88dc, entries=300, sequenceid=127, filesize=18.6 K 2024-11-15T22:17:26,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/8d69895111504ee0a7e9d49b6ff97d53 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/8d69895111504ee0a7e9d49b6ff97d53 2024-11-15T22:17:26,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/8d69895111504ee0a7e9d49b6ff97d53, entries=150, sequenceid=127, filesize=11.7 K 2024-11-15T22:17:26,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/6fbbe4423113448aa292ccdec51ef380 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/6fbbe4423113448aa292ccdec51ef380 2024-11-15T22:17:26,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/6fbbe4423113448aa292ccdec51ef380, entries=150, sequenceid=127, filesize=11.7 K 2024-11-15T22:17:26,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~49.20 KB/50380, heapSize ~126.34 KB/129376, currentSize=93.93 KB/96180 for 00af3e32e9b483f8e15f8f4708045315 in 96ms, sequenceid=127, compaction requested=true 2024-11-15T22:17:26,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:26,638 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:17:26,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:26,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:26,638 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:17:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:26,639 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52781 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:17:26,639 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/A is initiating minor compaction (all files) 2024-11-15T22:17:26,639 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/A in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:26,639 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/41d49b54f8c8451388365107dd93fa9a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/e800fef70de14940bd0f50035d7263a9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/eb874ee30cd245808299e6e3fe3d4c30, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/fa1c6a5b23954209bef61734f33c88dc] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=51.5 K 2024-11-15T22:17:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:26,639 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41d49b54f8c8451388365107dd93fa9a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731709042927 2024-11-15T22:17:26,639 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43421 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:17:26,642 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/B is initiating minor compaction (all files) 2024-11-15T22:17:26,642 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/B in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:26,642 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/36385214389c408abf4b78d422fe0172, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5adb63cf185e4146b61a5b30822872d5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c7da98e1a98447078b760730459075ed, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/8d69895111504ee0a7e9d49b6ff97d53] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=42.4 K 2024-11-15T22:17:26,642 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting e800fef70de14940bd0f50035d7263a9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731709043513 2024-11-15T22:17:26,643 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb874ee30cd245808299e6e3fe3d4c30, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731709044286 2024-11-15T22:17:26,643 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 36385214389c408abf4b78d422fe0172, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731709042927 2024-11-15T22:17:26,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:26,643 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa1c6a5b23954209bef61734f33c88dc, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731709046523 2024-11-15T22:17:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:26,643 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 5adb63cf185e4146b61a5b30822872d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731709043513 2024-11-15T22:17:26,644 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c7da98e1a98447078b760730459075ed, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731709044286 2024-11-15T22:17:26,645 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d69895111504ee0a7e9d49b6ff97d53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731709046523 2024-11-15T22:17:26,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=107.34 KB heapSize=295.92 KB 2024-11-15T22:17:26,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:26,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:26,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:26,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:26,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:26,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:26,655 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#A#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:26,656 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/db235dc41a114ec1a254cd61fe8eeb1f is 50, key is test_row_0/A:col10/1731709046539/Put/seqid=0 2024-11-15T22:17:26,679 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#B#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:26,687 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/9d323ffee1204732ba2f819dfe4a440b is 50, key is test_row_0/B:col10/1731709046539/Put/seqid=0 2024-11-15T22:17:26,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/8bb52cbcad2c4b97acb04bfff266c651 is 50, key is test_row_0/A:col10/1731709046632/Put/seqid=0 2024-11-15T22:17:26,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709106687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:26,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709106690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:26,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:26,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709106692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709106693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709106695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:26,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742281_1457 (size=12241) 2024-11-15T22:17:26,740 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/db235dc41a114ec1a254cd61fe8eeb1f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/db235dc41a114ec1a254cd61fe8eeb1f 2024-11-15T22:17:26,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742282_1458 (size=12241) 2024-11-15T22:17:26,766 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/A of 00af3e32e9b483f8e15f8f4708045315 into db235dc41a114ec1a254cd61fe8eeb1f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:26,766 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:26,766 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/A, priority=12, startTime=1731709046637; duration=0sec 2024-11-15T22:17:26,766 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:26,766 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:A 2024-11-15T22:17:26,766 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:17:26,771 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43421 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:17:26,771 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/C is initiating minor compaction (all files) 2024-11-15T22:17:26,771 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/C in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:26,771 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/024cbcb5324840a7bfa90e07370de7e4, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/38be5937d6464411ac1ab3f999d19d7e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/db376a74f70743f9b614d59bfd9da0f5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/6fbbe4423113448aa292ccdec51ef380] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=42.4 K 2024-11-15T22:17:26,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742283_1459 (size=16931) 2024-11-15T22:17:26,773 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 024cbcb5324840a7bfa90e07370de7e4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731709042927 2024-11-15T22:17:26,774 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38be5937d6464411ac1ab3f999d19d7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731709043513 2024-11-15T22:17:26,775 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting db376a74f70743f9b614d59bfd9da0f5, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731709044286 2024-11-15T22:17:26,776 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fbbe4423113448aa292ccdec51ef380, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731709046523 2024-11-15T22:17:26,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/8bb52cbcad2c4b97acb04bfff266c651 2024-11-15T22:17:26,787 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/9d323ffee1204732ba2f819dfe4a440b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/9d323ffee1204732ba2f819dfe4a440b 2024-11-15T22:17:26,799 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/B of 00af3e32e9b483f8e15f8f4708045315 into 9d323ffee1204732ba2f819dfe4a440b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:26,799 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:26,799 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/B, priority=12, startTime=1731709046638; duration=0sec 2024-11-15T22:17:26,799 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:26,799 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:B 2024-11-15T22:17:26,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/12d6dec3c0f041d9a485e1a89bb4616a is 50, key is test_row_0/B:col10/1731709046632/Put/seqid=0 2024-11-15T22:17:26,810 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#C#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:26,811 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/455b05f9a3474a9695f354d838db1f51 is 50, key is test_row_0/C:col10/1731709046539/Put/seqid=0 2024-11-15T22:17:26,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709106822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:26,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:26,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709106822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:26,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:26,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709106822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:26,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709106822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709106822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:26,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742285_1461 (size=12241) 2024-11-15T22:17:26,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742284_1460 (size=12151) 2024-11-15T22:17:26,858 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/455b05f9a3474a9695f354d838db1f51 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/455b05f9a3474a9695f354d838db1f51 2024-11-15T22:17:26,864 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/C of 00af3e32e9b483f8e15f8f4708045315 into 455b05f9a3474a9695f354d838db1f51(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:26,864 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:26,864 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/C, priority=12, startTime=1731709046639; duration=0sec 2024-11-15T22:17:26,864 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:26,864 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:C 2024-11-15T22:17:27,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709107042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:27,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:27,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709107042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:27,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:27,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709107042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:27,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:27,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709107042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:27,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709107042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:27,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/12d6dec3c0f041d9a485e1a89bb4616a 2024-11-15T22:17:27,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/a8a4896873ac4cd583783691b9dcda74 is 50, key is test_row_0/C:col10/1731709046632/Put/seqid=0 2024-11-15T22:17:27,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742286_1462 (size=12151) 2024-11-15T22:17:27,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709107364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:27,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:27,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709107365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:27,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:27,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709107365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:27,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:27,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709107365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:27,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709107366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:27,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/a8a4896873ac4cd583783691b9dcda74 2024-11-15T22:17:27,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/8bb52cbcad2c4b97acb04bfff266c651 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/8bb52cbcad2c4b97acb04bfff266c651 2024-11-15T22:17:27,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/8bb52cbcad2c4b97acb04bfff266c651, entries=250, sequenceid=158, filesize=16.5 K 2024-11-15T22:17:27,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/12d6dec3c0f041d9a485e1a89bb4616a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/12d6dec3c0f041d9a485e1a89bb4616a 2024-11-15T22:17:27,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/12d6dec3c0f041d9a485e1a89bb4616a, entries=150, sequenceid=158, filesize=11.9 K 2024-11-15T22:17:27,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/a8a4896873ac4cd583783691b9dcda74 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a8a4896873ac4cd583783691b9dcda74 2024-11-15T22:17:27,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a8a4896873ac4cd583783691b9dcda74, entries=150, sequenceid=158, filesize=11.9 K 2024-11-15T22:17:27,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~295.88 KB/302976, currentSize=73.80 KB/75570 for 00af3e32e9b483f8e15f8f4708045315 in 1171ms, sequenceid=158, compaction requested=false 2024-11-15T22:17:27,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:27,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T22:17:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:27,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:27,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:27,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:27,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:27,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:27,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:27,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/722fd32b7fed419d900317609afd7d25 is 50, key is test_row_0/A:col10/1731709046690/Put/seqid=0 2024-11-15T22:17:27,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742287_1463 (size=14541) 2024-11-15T22:17:27,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709107937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:27,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709107937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:27,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:27,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709107945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:27,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709107945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:27,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709107949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709108062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:28,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709108068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:28,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709108071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:28,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709108072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:28,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709108076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709108282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:28,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709108292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:28,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709108292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709108304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:28,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709108314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:28,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/722fd32b7fed419d900317609afd7d25 2024-11-15T22:17:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T22:17:28,390 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:28,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:17:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-15T22:17:28,393 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:17:28,394 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:17:28,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:17:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T22:17:28,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/a6860249aae648cb94de6dd35d071385 is 50, key is test_row_0/B:col10/1731709046690/Put/seqid=0 2024-11-15T22:17:28,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742288_1464 (size=12151) 2024-11-15T22:17:28,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/a6860249aae648cb94de6dd35d071385 2024-11-15T22:17:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T22:17:28,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/2afafaa6264c45a1b27fd15e3fb30ce9 is 50, key is test_row_0/C:col10/1731709046690/Put/seqid=0 2024-11-15T22:17:28,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T22:17:28,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:28,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:28,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:28,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:28,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:28,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742289_1465 (size=12151) 2024-11-15T22:17:28,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/2afafaa6264c45a1b27fd15e3fb30ce9 2024-11-15T22:17:28,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/722fd32b7fed419d900317609afd7d25 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/722fd32b7fed419d900317609afd7d25 2024-11-15T22:17:28,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/722fd32b7fed419d900317609afd7d25, entries=200, sequenceid=178, filesize=14.2 K 2024-11-15T22:17:28,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/a6860249aae648cb94de6dd35d071385 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/a6860249aae648cb94de6dd35d071385 2024-11-15T22:17:28,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709108613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:28,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709108625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709108629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/a6860249aae648cb94de6dd35d071385, entries=150, sequenceid=178, filesize=11.9 K 2024-11-15T22:17:28,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/2afafaa6264c45a1b27fd15e3fb30ce9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/2afafaa6264c45a1b27fd15e3fb30ce9 2024-11-15T22:17:28,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709108632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:28,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709108632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:28,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:28,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:28,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/2afafaa6264c45a1b27fd15e3fb30ce9, entries=150, sequenceid=178, filesize=11.9 K 2024-11-15T22:17:28,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 00af3e32e9b483f8e15f8f4708045315 in 766ms, sequenceid=178, compaction requested=true 2024-11-15T22:17:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:17:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T22:17:28,664 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:28,664 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:28,665 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43713 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:28,665 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/A is initiating minor compaction (all files) 2024-11-15T22:17:28,666 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/A in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:28,666 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/db235dc41a114ec1a254cd61fe8eeb1f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/8bb52cbcad2c4b97acb04bfff266c651, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/722fd32b7fed419d900317609afd7d25] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=42.7 K 2024-11-15T22:17:28,666 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:28,666 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/C is initiating minor compaction (all files) 2024-11-15T22:17:28,666 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/C in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:28,666 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/455b05f9a3474a9695f354d838db1f51, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a8a4896873ac4cd583783691b9dcda74, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/2afafaa6264c45a1b27fd15e3fb30ce9] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=35.7 K 2024-11-15T22:17:28,666 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting db235dc41a114ec1a254cd61fe8eeb1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731709046523 2024-11-15T22:17:28,667 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 455b05f9a3474a9695f354d838db1f51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731709046523 2024-11-15T22:17:28,667 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bb52cbcad2c4b97acb04bfff266c651, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731709046627 2024-11-15T22:17:28,667 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting a8a4896873ac4cd583783691b9dcda74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731709046632 2024-11-15T22:17:28,667 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 722fd32b7fed419d900317609afd7d25, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731709046687 2024-11-15T22:17:28,668 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 2afafaa6264c45a1b27fd15e3fb30ce9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731709046687 2024-11-15T22:17:28,686 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#A#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:28,687 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/a24f4270692c41da8e912a56f56e69e2 is 50, key is test_row_0/A:col10/1731709046690/Put/seqid=0 2024-11-15T22:17:28,694 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#C#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:28,695 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/5f159bf16a7741a480568d465cee4735 is 50, key is test_row_0/C:col10/1731709046690/Put/seqid=0 2024-11-15T22:17:28,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T22:17:28,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:28,710 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T22:17:28,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:28,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:28,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:28,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:28,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:28,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T22:17:28,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/f2bafdc6fa8640bab286936226459d50 is 50, key is test_row_0/A:col10/1731709047943/Put/seqid=0 2024-11-15T22:17:28,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742290_1466 (size=12493) 2024-11-15T22:17:28,743 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/a24f4270692c41da8e912a56f56e69e2 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a24f4270692c41da8e912a56f56e69e2 2024-11-15T22:17:28,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742291_1467 (size=12493) 2024-11-15T22:17:28,756 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/A of 00af3e32e9b483f8e15f8f4708045315 into a24f4270692c41da8e912a56f56e69e2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:28,756 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:28,756 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/A, priority=13, startTime=1731709048664; duration=0sec 2024-11-15T22:17:28,756 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:28,756 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:A 2024-11-15T22:17:28,757 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:28,762 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:28,762 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/B is initiating minor compaction (all files) 2024-11-15T22:17:28,762 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/B in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:28,762 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/9d323ffee1204732ba2f819dfe4a440b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/12d6dec3c0f041d9a485e1a89bb4616a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/a6860249aae648cb94de6dd35d071385] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=35.7 K 2024-11-15T22:17:28,762 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d323ffee1204732ba2f819dfe4a440b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731709046523 2024-11-15T22:17:28,763 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12d6dec3c0f041d9a485e1a89bb4616a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731709046632 2024-11-15T22:17:28,763 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6860249aae648cb94de6dd35d071385, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731709046687 2024-11-15T22:17:28,774 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:28,775 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/1011fd5361434ef597a4d004288e5c67 is 50, key is test_row_0/B:col10/1731709046690/Put/seqid=0 2024-11-15T22:17:28,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742292_1468 (size=12151) 2024-11-15T22:17:28,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742293_1469 (size=12493) 2024-11-15T22:17:28,838 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/1011fd5361434ef597a4d004288e5c67 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/1011fd5361434ef597a4d004288e5c67 2024-11-15T22:17:28,844 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/B of 00af3e32e9b483f8e15f8f4708045315 into 1011fd5361434ef597a4d004288e5c67(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:28,844 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:28,844 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/B, priority=13, startTime=1731709048664; duration=0sec 2024-11-15T22:17:28,844 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:28,844 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:B 2024-11-15T22:17:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T22:17:29,162 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/5f159bf16a7741a480568d465cee4735 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/5f159bf16a7741a480568d465cee4735 2024-11-15T22:17:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:29,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:29,168 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/C of 00af3e32e9b483f8e15f8f4708045315 into 5f159bf16a7741a480568d465cee4735(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:29,169 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:29,169 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/C, priority=13, startTime=1731709048664; duration=0sec 2024-11-15T22:17:29,169 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:29,169 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:C 2024-11-15T22:17:29,189 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/f2bafdc6fa8640bab286936226459d50 2024-11-15T22:17:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/5fd6395e0c374d7fb709661caf038e10 is 50, key is test_row_0/B:col10/1731709047943/Put/seqid=0 2024-11-15T22:17:29,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709109198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709109200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709109202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709109202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709109210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742294_1470 (size=12151) 2024-11-15T22:17:29,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709109322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709109322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709109332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709109332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709109332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T22:17:29,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709109541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709109542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709109542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709109542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709109542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,639 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/5fd6395e0c374d7fb709661caf038e10 2024-11-15T22:17:29,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/81b0c49a07fd434e88cd09caae26384e is 50, key is test_row_0/C:col10/1731709047943/Put/seqid=0 2024-11-15T22:17:29,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742295_1471 (size=12151) 2024-11-15T22:17:29,754 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f10458dd6a7d4929aaf55e6906a5e318, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/3bd39ed63353417080b5d99aae909cac, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/41d49b54f8c8451388365107dd93fa9a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/1942fc4e9c164a06af9ed8686ea1bd13, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/e800fef70de14940bd0f50035d7263a9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/eb874ee30cd245808299e6e3fe3d4c30, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/fa1c6a5b23954209bef61734f33c88dc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/db235dc41a114ec1a254cd61fe8eeb1f, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/8bb52cbcad2c4b97acb04bfff266c651, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/722fd32b7fed419d900317609afd7d25] to archive 2024-11-15T22:17:29,755 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:17:29,756 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f10458dd6a7d4929aaf55e6906a5e318 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f10458dd6a7d4929aaf55e6906a5e318 2024-11-15T22:17:29,757 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/3bd39ed63353417080b5d99aae909cac to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/3bd39ed63353417080b5d99aae909cac 2024-11-15T22:17:29,764 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/41d49b54f8c8451388365107dd93fa9a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/41d49b54f8c8451388365107dd93fa9a 2024-11-15T22:17:29,766 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/1942fc4e9c164a06af9ed8686ea1bd13 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/1942fc4e9c164a06af9ed8686ea1bd13 2024-11-15T22:17:29,769 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/e800fef70de14940bd0f50035d7263a9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/e800fef70de14940bd0f50035d7263a9 2024-11-15T22:17:29,770 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/eb874ee30cd245808299e6e3fe3d4c30 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/eb874ee30cd245808299e6e3fe3d4c30 2024-11-15T22:17:29,771 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/fa1c6a5b23954209bef61734f33c88dc to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/fa1c6a5b23954209bef61734f33c88dc 2024-11-15T22:17:29,772 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/db235dc41a114ec1a254cd61fe8eeb1f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/db235dc41a114ec1a254cd61fe8eeb1f 2024-11-15T22:17:29,773 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/8bb52cbcad2c4b97acb04bfff266c651 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/8bb52cbcad2c4b97acb04bfff266c651 2024-11-15T22:17:29,774 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/722fd32b7fed419d900317609afd7d25 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/722fd32b7fed419d900317609afd7d25 2024-11-15T22:17:29,776 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/34b9f0724dd748a08ec524f834b23eb6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/d4262c8777504c8aa3027b3314f7cbdf, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/36385214389c408abf4b78d422fe0172, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c7c21bc3078740ddbd61bfda3a9be77a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5adb63cf185e4146b61a5b30822872d5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c7da98e1a98447078b760730459075ed, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/9d323ffee1204732ba2f819dfe4a440b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/8d69895111504ee0a7e9d49b6ff97d53, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/12d6dec3c0f041d9a485e1a89bb4616a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/a6860249aae648cb94de6dd35d071385] to archive 2024-11-15T22:17:29,777 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:17:29,781 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/34b9f0724dd748a08ec524f834b23eb6 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/34b9f0724dd748a08ec524f834b23eb6 2024-11-15T22:17:29,783 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/d4262c8777504c8aa3027b3314f7cbdf to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/d4262c8777504c8aa3027b3314f7cbdf 2024-11-15T22:17:29,784 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/36385214389c408abf4b78d422fe0172 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/36385214389c408abf4b78d422fe0172 2024-11-15T22:17:29,785 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c7c21bc3078740ddbd61bfda3a9be77a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c7c21bc3078740ddbd61bfda3a9be77a 2024-11-15T22:17:29,788 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5adb63cf185e4146b61a5b30822872d5 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5adb63cf185e4146b61a5b30822872d5 2024-11-15T22:17:29,794 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c7da98e1a98447078b760730459075ed to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c7da98e1a98447078b760730459075ed 2024-11-15T22:17:29,795 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/9d323ffee1204732ba2f819dfe4a440b to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/9d323ffee1204732ba2f819dfe4a440b 2024-11-15T22:17:29,796 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/8d69895111504ee0a7e9d49b6ff97d53 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/8d69895111504ee0a7e9d49b6ff97d53 2024-11-15T22:17:29,797 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/12d6dec3c0f041d9a485e1a89bb4616a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/12d6dec3c0f041d9a485e1a89bb4616a 2024-11-15T22:17:29,811 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/a6860249aae648cb94de6dd35d071385 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/a6860249aae648cb94de6dd35d071385 2024-11-15T22:17:29,815 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/40dc6ac3d8664ac69ed4eed4abbbbc18, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/bb6772f854454553bedf147fb4416dcc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/024cbcb5324840a7bfa90e07370de7e4, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/7519b20c311643f4b7337073a22c5be3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/38be5937d6464411ac1ab3f999d19d7e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/db376a74f70743f9b614d59bfd9da0f5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/455b05f9a3474a9695f354d838db1f51, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/6fbbe4423113448aa292ccdec51ef380, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a8a4896873ac4cd583783691b9dcda74, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/2afafaa6264c45a1b27fd15e3fb30ce9] to archive 2024-11-15T22:17:29,817 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:17:29,822 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/40dc6ac3d8664ac69ed4eed4abbbbc18 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/40dc6ac3d8664ac69ed4eed4abbbbc18 2024-11-15T22:17:29,822 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/bb6772f854454553bedf147fb4416dcc to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/bb6772f854454553bedf147fb4416dcc 2024-11-15T22:17:29,823 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/024cbcb5324840a7bfa90e07370de7e4 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/024cbcb5324840a7bfa90e07370de7e4 2024-11-15T22:17:29,824 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/7519b20c311643f4b7337073a22c5be3 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/7519b20c311643f4b7337073a22c5be3 2024-11-15T22:17:29,825 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/38be5937d6464411ac1ab3f999d19d7e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/38be5937d6464411ac1ab3f999d19d7e 2024-11-15T22:17:29,829 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/db376a74f70743f9b614d59bfd9da0f5 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/db376a74f70743f9b614d59bfd9da0f5 2024-11-15T22:17:29,833 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/455b05f9a3474a9695f354d838db1f51 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/455b05f9a3474a9695f354d838db1f51 2024-11-15T22:17:29,834 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/6fbbe4423113448aa292ccdec51ef380 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/6fbbe4423113448aa292ccdec51ef380 2024-11-15T22:17:29,835 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a8a4896873ac4cd583783691b9dcda74 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a8a4896873ac4cd583783691b9dcda74 2024-11-15T22:17:29,836 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/2afafaa6264c45a1b27fd15e3fb30ce9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/2afafaa6264c45a1b27fd15e3fb30ce9 2024-11-15T22:17:29,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709109861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709109861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709109862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709109862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:29,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709109862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:29,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:29,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,082 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/81b0c49a07fd434e88cd09caae26384e 2024-11-15T22:17:30,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/f2bafdc6fa8640bab286936226459d50 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f2bafdc6fa8640bab286936226459d50 2024-11-15T22:17:30,105 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f2bafdc6fa8640bab286936226459d50, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T22:17:30,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/5fd6395e0c374d7fb709661caf038e10 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5fd6395e0c374d7fb709661caf038e10 2024-11-15T22:17:30,119 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5fd6395e0c374d7fb709661caf038e10, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T22:17:30,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/81b0c49a07fd434e88cd09caae26384e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/81b0c49a07fd434e88cd09caae26384e 2024-11-15T22:17:30,130 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/81b0c49a07fd434e88cd09caae26384e, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T22:17:30,131 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 00af3e32e9b483f8e15f8f4708045315 in 1421ms, sequenceid=193, compaction requested=false 2024-11-15T22:17:30,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:30,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:30,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-15T22:17:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-15T22:17:30,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-15T22:17:30,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7390 sec 2024-11-15T22:17:30,137 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.7440 sec 2024-11-15T22:17:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:30,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T22:17:30,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:30,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:30,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:30,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:30,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:30,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:30,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/03df7b08a49b49c5a5b83e2bcdd49493 is 50, key is test_row_0/A:col10/1731709049197/Put/seqid=0 2024-11-15T22:17:30,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742296_1472 (size=14541) 2024-11-15T22:17:30,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709110398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:30,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709110403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709110405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709110409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709110411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709110522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709110523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:30,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T22:17:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709110531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:30,542 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:30,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709110531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709110531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:17:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-15T22:17:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T22:17:30,546 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:17:30,546 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:17:30,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:17:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T22:17:30,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T22:17:30,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:30,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:30,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:30,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:30,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:30,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709110741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:30,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709110741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709110751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:30,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709110751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709110751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:30,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/03df7b08a49b49c5a5b83e2bcdd49493 2024-11-15T22:17:30,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/045591a206684de2bdd3b17e737f43ed is 50, key is test_row_0/B:col10/1731709049197/Put/seqid=0 2024-11-15T22:17:30,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742297_1473 (size=12151) 2024-11-15T22:17:30,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/045591a206684de2bdd3b17e737f43ed 2024-11-15T22:17:30,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T22:17:30,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:30,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:30,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:30,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:30,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:30,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/9ab86da33aad4d77816864f68b0e4500 is 50, key is test_row_0/C:col10/1731709049197/Put/seqid=0 2024-11-15T22:17:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T22:17:30,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742298_1474 (size=12151) 2024-11-15T22:17:30,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/9ab86da33aad4d77816864f68b0e4500 2024-11-15T22:17:30,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/03df7b08a49b49c5a5b83e2bcdd49493 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/03df7b08a49b49c5a5b83e2bcdd49493 2024-11-15T22:17:30,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/03df7b08a49b49c5a5b83e2bcdd49493, entries=200, sequenceid=215, filesize=14.2 K 2024-11-15T22:17:30,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/045591a206684de2bdd3b17e737f43ed as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/045591a206684de2bdd3b17e737f43ed 2024-11-15T22:17:30,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/045591a206684de2bdd3b17e737f43ed, entries=150, sequenceid=215, filesize=11.9 K 2024-11-15T22:17:30,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/9ab86da33aad4d77816864f68b0e4500 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/9ab86da33aad4d77816864f68b0e4500 2024-11-15T22:17:30,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/9ab86da33aad4d77816864f68b0e4500, entries=150, sequenceid=215, filesize=11.9 K 2024-11-15T22:17:30,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 00af3e32e9b483f8e15f8f4708045315 in 509ms, sequenceid=215, compaction requested=true 2024-11-15T22:17:30,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:30,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:30,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:30,896 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:30,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:30,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:30,896 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:30,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:30,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:30,896 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:30,896 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:30,897 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/A is initiating minor compaction (all files) 2024-11-15T22:17:30,897 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/B is initiating minor compaction (all files) 2024-11-15T22:17:30,897 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/B in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:30,897 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/A in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:30,897 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a24f4270692c41da8e912a56f56e69e2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f2bafdc6fa8640bab286936226459d50, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/03df7b08a49b49c5a5b83e2bcdd49493] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=38.3 K 2024-11-15T22:17:30,897 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/1011fd5361434ef597a4d004288e5c67, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5fd6395e0c374d7fb709661caf038e10, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/045591a206684de2bdd3b17e737f43ed] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=35.9 K 2024-11-15T22:17:30,897 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting a24f4270692c41da8e912a56f56e69e2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731709046687 2024-11-15T22:17:30,897 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 1011fd5361434ef597a4d004288e5c67, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731709046687 2024-11-15T22:17:30,897 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2bafdc6fa8640bab286936226459d50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731709047930 2024-11-15T22:17:30,897 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fd6395e0c374d7fb709661caf038e10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731709047930 2024-11-15T22:17:30,897 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 045591a206684de2bdd3b17e737f43ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731709049197 2024-11-15T22:17:30,897 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03df7b08a49b49c5a5b83e2bcdd49493, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731709049197 2024-11-15T22:17:30,907 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#A#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:30,907 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/e2c861c97e6d405f8721dc5816d03552 is 50, key is test_row_0/A:col10/1731709049197/Put/seqid=0 2024-11-15T22:17:30,914 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#B#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:30,915 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/365fd7753ce04100b46d9522a03838e5 is 50, key is test_row_0/B:col10/1731709049197/Put/seqid=0 2024-11-15T22:17:30,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742299_1475 (size=12254) 2024-11-15T22:17:30,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742300_1476 (size=12254) 2024-11-15T22:17:30,937 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/e2c861c97e6d405f8721dc5816d03552 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/e2c861c97e6d405f8721dc5816d03552 2024-11-15T22:17:30,940 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/365fd7753ce04100b46d9522a03838e5 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/365fd7753ce04100b46d9522a03838e5 2024-11-15T22:17:30,947 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/A of 00af3e32e9b483f8e15f8f4708045315 into e2c861c97e6d405f8721dc5816d03552(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:30,947 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:30,947 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/A, priority=13, startTime=1731709050896; duration=0sec 2024-11-15T22:17:30,947 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:30,947 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:A 2024-11-15T22:17:30,947 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:30,948 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:30,948 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/C is initiating minor compaction (all files) 2024-11-15T22:17:30,948 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/C in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:30,948 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/5f159bf16a7741a480568d465cee4735, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/81b0c49a07fd434e88cd09caae26384e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/9ab86da33aad4d77816864f68b0e4500] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=35.9 K 2024-11-15T22:17:30,948 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f159bf16a7741a480568d465cee4735, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731709046687 2024-11-15T22:17:30,949 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81b0c49a07fd434e88cd09caae26384e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731709047930 2024-11-15T22:17:30,949 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ab86da33aad4d77816864f68b0e4500, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731709049197 2024-11-15T22:17:30,951 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/B of 00af3e32e9b483f8e15f8f4708045315 into 365fd7753ce04100b46d9522a03838e5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:30,951 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:30,951 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/B, priority=13, startTime=1731709050896; duration=0sec 2024-11-15T22:17:30,951 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:30,951 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:B 2024-11-15T22:17:30,957 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#C#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:30,957 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/7b9304b1859844fcad93ece6375d9bb2 is 50, key is test_row_0/C:col10/1731709049197/Put/seqid=0 2024-11-15T22:17:31,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T22:17:31,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:31,004 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T22:17:31,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:31,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:31,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:31,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:31,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:31,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:31,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/771d581b9d964062bee4e0cc2414d40b is 50, key is test_row_0/A:col10/1731709050395/Put/seqid=0 2024-11-15T22:17:31,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742301_1477 (size=12254) 2024-11-15T22:17:31,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742302_1478 (size=12151) 2024-11-15T22:17:31,041 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/771d581b9d964062bee4e0cc2414d40b 2024-11-15T22:17:31,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/5543934eede446118d04d85037bc8755 is 50, key is test_row_0/B:col10/1731709050395/Put/seqid=0 2024-11-15T22:17:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:31,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:31,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742303_1479 (size=12151) 2024-11-15T22:17:31,061 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/5543934eede446118d04d85037bc8755 2024-11-15T22:17:31,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/3c480bedb22d432fa387de393c9a6708 is 50, key is test_row_0/C:col10/1731709050395/Put/seqid=0 2024-11-15T22:17:31,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742304_1480 (size=12151) 2024-11-15T22:17:31,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709111087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709111088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709111095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709111097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709111097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T22:17:31,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709111202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709111202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709111202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709111212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709111212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,423 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/7b9304b1859844fcad93ece6375d9bb2 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/7b9304b1859844fcad93ece6375d9bb2 2024-11-15T22:17:31,426 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/C of 00af3e32e9b483f8e15f8f4708045315 into 7b9304b1859844fcad93ece6375d9bb2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:31,426 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:31,426 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/C, priority=13, startTime=1731709050896; duration=0sec 2024-11-15T22:17:31,426 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:31,426 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:C 2024-11-15T22:17:31,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709111421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709111421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709111421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709111421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709111421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,478 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/3c480bedb22d432fa387de393c9a6708 2024-11-15T22:17:31,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/771d581b9d964062bee4e0cc2414d40b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/771d581b9d964062bee4e0cc2414d40b 2024-11-15T22:17:31,485 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/771d581b9d964062bee4e0cc2414d40b, entries=150, sequenceid=231, filesize=11.9 K 2024-11-15T22:17:31,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/5543934eede446118d04d85037bc8755 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5543934eede446118d04d85037bc8755 2024-11-15T22:17:31,488 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5543934eede446118d04d85037bc8755, entries=150, sequenceid=231, filesize=11.9 K 2024-11-15T22:17:31,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/3c480bedb22d432fa387de393c9a6708 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/3c480bedb22d432fa387de393c9a6708 2024-11-15T22:17:31,492 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/3c480bedb22d432fa387de393c9a6708, entries=150, sequenceid=231, filesize=11.9 K 2024-11-15T22:17:31,493 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 00af3e32e9b483f8e15f8f4708045315 in 489ms, sequenceid=231, compaction requested=false 2024-11-15T22:17:31,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:31,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:31,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-15T22:17:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-15T22:17:31,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-15T22:17:31,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 948 msec 2024-11-15T22:17:31,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 952 msec 2024-11-15T22:17:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T22:17:31,682 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:31,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:17:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-15T22:17:31,684 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:17:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T22:17:31,684 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:17:31,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:17:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:31,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731709051082, max timestamp=1731709051736Num uniques 859; 2024-11-15T22:17:31,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:17:31,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=105.11 KB heapSize=307.03 KB 2024-11-15T22:17:31,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731709051082, max timestamp=1731709051736Num uniques 859; 2024-11-15T22:17:31,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:17:31,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:31,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:31,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:31,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:31,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:31,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:31,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/854d88e0bd204d479c6c7ad0612b7f35 is 50, key is test_row_0/A:col10/1731709051082/Put/seqid=0 2024-11-15T22:17:31,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742305_1481 (size=16931) 2024-11-15T22:17:31,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709111772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709111773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T22:17:31,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709111786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709111788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709111792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T22:17:31,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:31,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:31,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:31,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:31,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:31,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709111891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709111891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709111912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709111912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709111912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:31,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T22:17:31,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:31,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:31,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:31,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:31,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:31,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:32,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T22:17:32,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:32,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709112111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:32,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:32,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709112111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:32,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709112122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:32,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709112131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:32,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709112131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:32,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T22:17:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:32,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:32,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/854d88e0bd204d479c6c7ad0612b7f35 2024-11-15T22:17:32,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/e839b8797a0d4a758e97cd8c33c73674 is 50, key is test_row_0/B:col10/1731709051082/Put/seqid=0 2024-11-15T22:17:32,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742306_1482 (size=12151) 2024-11-15T22:17:32,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T22:17:32,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:32,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:32,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:32,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:32,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:32,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:32,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T22:17:32,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709112422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:32,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:32,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709112431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:32,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:32,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709112441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:32,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709112441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709112441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:32,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T22:17:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:32,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:32,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T22:17:32,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:32,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:32,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:32,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:32,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:32,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:32,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/e839b8797a0d4a758e97cd8c33c73674 2024-11-15T22:17:32,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/c18f8810557a48cb819f4d62c7ddade6 is 50, key is test_row_0/C:col10/1731709051082/Put/seqid=0 2024-11-15T22:17:32,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742307_1483 (size=12151) 2024-11-15T22:17:32,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/c18f8810557a48cb819f4d62c7ddade6 2024-11-15T22:17:32,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/854d88e0bd204d479c6c7ad0612b7f35 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/854d88e0bd204d479c6c7ad0612b7f35 2024-11-15T22:17:32,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/854d88e0bd204d479c6c7ad0612b7f35, entries=250, sequenceid=255, filesize=16.5 K 2024-11-15T22:17:32,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/e839b8797a0d4a758e97cd8c33c73674 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/e839b8797a0d4a758e97cd8c33c73674 2024-11-15T22:17:32,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/e839b8797a0d4a758e97cd8c33c73674, entries=150, sequenceid=255, filesize=11.9 K 2024-11-15T22:17:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/c18f8810557a48cb819f4d62c7ddade6 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/c18f8810557a48cb819f4d62c7ddade6 2024-11-15T22:17:32,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/c18f8810557a48cb819f4d62c7ddade6, entries=150, sequenceid=255, filesize=11.9 K 2024-11-15T22:17:32,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~76.04 KB/77860, heapSize ~214.13 KB/219264, currentSize=100.63 KB/103050 for 00af3e32e9b483f8e15f8f4708045315 in 999ms, sequenceid=255, compaction requested=true 2024-11-15T22:17:32,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:32,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:32,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:32,740 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:32,740 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:32,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:32,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:32,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:32,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:32,741 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41336 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:32,741 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36556 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:32,741 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/B is initiating minor compaction (all files) 2024-11-15T22:17:32,741 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/A is initiating minor compaction (all files) 2024-11-15T22:17:32,741 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/A in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:32,741 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/B in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:32,741 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/e2c861c97e6d405f8721dc5816d03552, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/771d581b9d964062bee4e0cc2414d40b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/854d88e0bd204d479c6c7ad0612b7f35] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=40.4 K 2024-11-15T22:17:32,741 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/365fd7753ce04100b46d9522a03838e5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5543934eede446118d04d85037bc8755, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/e839b8797a0d4a758e97cd8c33c73674] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=35.7 K 2024-11-15T22:17:32,741 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2c861c97e6d405f8721dc5816d03552, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731709049197 2024-11-15T22:17:32,741 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 365fd7753ce04100b46d9522a03838e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731709049197 2024-11-15T22:17:32,741 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 771d581b9d964062bee4e0cc2414d40b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731709050395 2024-11-15T22:17:32,741 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 5543934eede446118d04d85037bc8755, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731709050395 2024-11-15T22:17:32,742 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 854d88e0bd204d479c6c7ad0612b7f35, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731709051082 2024-11-15T22:17:32,742 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting e839b8797a0d4a758e97cd8c33c73674, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731709051082 2024-11-15T22:17:32,749 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:32,749 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/9f7ec0d00ce941b599fd33544dba68c9 is 50, key is test_row_0/B:col10/1731709051082/Put/seqid=0 2024-11-15T22:17:32,752 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#A#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:32,753 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/a334533295db407ca426062c2e31a52a is 50, key is test_row_0/A:col10/1731709051082/Put/seqid=0 2024-11-15T22:17:32,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742308_1484 (size=12357) 2024-11-15T22:17:32,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T22:17:32,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:32,762 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T22:17:32,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:32,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:32,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:32,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:32,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:32,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:32,763 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/9f7ec0d00ce941b599fd33544dba68c9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/9f7ec0d00ce941b599fd33544dba68c9 2024-11-15T22:17:32,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742309_1485 (size=12357) 2024-11-15T22:17:32,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/4bc9636ec1ed4e9f92fd3e21efadc863 is 50, key is test_row_0/A:col10/1731709051787/Put/seqid=0 2024-11-15T22:17:32,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742310_1486 (size=12301) 2024-11-15T22:17:32,778 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/4bc9636ec1ed4e9f92fd3e21efadc863 2024-11-15T22:17:32,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/0a4f4410da394877afec08c995f595c8 is 50, key is test_row_0/B:col10/1731709051787/Put/seqid=0 2024-11-15T22:17:32,790 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/B of 00af3e32e9b483f8e15f8f4708045315 into 9f7ec0d00ce941b599fd33544dba68c9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:32,790 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:32,790 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/B, priority=13, startTime=1731709052740; duration=0sec 2024-11-15T22:17:32,790 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:32,790 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:B 2024-11-15T22:17:32,790 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:32,794 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36556 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:32,794 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/C is initiating minor compaction (all files) 2024-11-15T22:17:32,794 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/C in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:32,794 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/7b9304b1859844fcad93ece6375d9bb2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/3c480bedb22d432fa387de393c9a6708, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/c18f8810557a48cb819f4d62c7ddade6] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=35.7 K 2024-11-15T22:17:32,794 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b9304b1859844fcad93ece6375d9bb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731709049197 2024-11-15T22:17:32,795 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c480bedb22d432fa387de393c9a6708, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731709050395 2024-11-15T22:17:32,795 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c18f8810557a48cb819f4d62c7ddade6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731709051082 2024-11-15T22:17:32,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742311_1487 (size=12301) 2024-11-15T22:17:32,803 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/0a4f4410da394877afec08c995f595c8 2024-11-15T22:17:32,819 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#C#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:32,819 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/ffb964f0814846a188e23c5abce985d7 is 50, key is test_row_0/C:col10/1731709051082/Put/seqid=0 2024-11-15T22:17:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T22:17:32,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/ce7fde65df4446e3a3482a7c47d1b225 is 50, key is test_row_0/C:col10/1731709051787/Put/seqid=0 2024-11-15T22:17:32,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742312_1488 (size=12357) 2024-11-15T22:17:32,829 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/ffb964f0814846a188e23c5abce985d7 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ffb964f0814846a188e23c5abce985d7 2024-11-15T22:17:32,834 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/C of 00af3e32e9b483f8e15f8f4708045315 into ffb964f0814846a188e23c5abce985d7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:32,835 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:32,835 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/C, priority=13, startTime=1731709052740; duration=0sec 2024-11-15T22:17:32,835 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:32,835 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:C 2024-11-15T22:17:32,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742313_1489 (size=12301) 2024-11-15T22:17:32,860 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/ce7fde65df4446e3a3482a7c47d1b225 2024-11-15T22:17:32,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/4bc9636ec1ed4e9f92fd3e21efadc863 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/4bc9636ec1ed4e9f92fd3e21efadc863 2024-11-15T22:17:32,879 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/4bc9636ec1ed4e9f92fd3e21efadc863, entries=150, sequenceid=273, filesize=12.0 K 2024-11-15T22:17:32,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/0a4f4410da394877afec08c995f595c8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/0a4f4410da394877afec08c995f595c8 2024-11-15T22:17:32,890 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/0a4f4410da394877afec08c995f595c8, entries=150, sequenceid=273, filesize=12.0 K 2024-11-15T22:17:32,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/ce7fde65df4446e3a3482a7c47d1b225 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ce7fde65df4446e3a3482a7c47d1b225 2024-11-15T22:17:32,896 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ce7fde65df4446e3a3482a7c47d1b225, entries=150, sequenceid=273, filesize=12.0 K 2024-11-15T22:17:32,899 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 00af3e32e9b483f8e15f8f4708045315 in 137ms, sequenceid=273, compaction requested=false 2024-11-15T22:17:32,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:32,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:32,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-15T22:17:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-15T22:17:32,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-15T22:17:32,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2180 sec 2024-11-15T22:17:32,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.2240 sec 2024-11-15T22:17:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:32,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T22:17:32,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:32,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:32,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:32,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:32,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:32,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:32,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/67556d93c7784746b59f586ed098fa51 is 50, key is test_row_0/A:col10/1731709052972/Put/seqid=0 2024-11-15T22:17:33,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742314_1490 (size=19621) 2024-11-15T22:17:33,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/67556d93c7784746b59f586ed098fa51 2024-11-15T22:17:33,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/4be9e2ded85c40428d1cafa89f29702d is 50, key is test_row_0/B:col10/1731709052972/Put/seqid=0 2024-11-15T22:17:33,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742315_1491 (size=12301) 2024-11-15T22:17:33,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/4be9e2ded85c40428d1cafa89f29702d 2024-11-15T22:17:33,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/a4fd6617e989408ba3c64b4916f488bd is 50, key is test_row_0/C:col10/1731709052972/Put/seqid=0 2024-11-15T22:17:33,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731709052999, max timestamp=1731709053037Num uniques 859; 2024-11-15T22:17:33,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:17:33,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731709052999, max timestamp=1731709053037Num uniques 859; 2024-11-15T22:17:33,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:17:33,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731709052999, max timestamp=1731709053037Num uniques 859; 2024-11-15T22:17:33,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:17:33,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742316_1492 (size=12301) 2024-11-15T22:17:33,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/a4fd6617e989408ba3c64b4916f488bd 2024-11-15T22:17:33,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/67556d93c7784746b59f586ed098fa51 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/67556d93c7784746b59f586ed098fa51 2024-11-15T22:17:33,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/67556d93c7784746b59f586ed098fa51, entries=300, sequenceid=286, filesize=19.2 K 2024-11-15T22:17:33,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/4be9e2ded85c40428d1cafa89f29702d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/4be9e2ded85c40428d1cafa89f29702d 2024-11-15T22:17:33,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/4be9e2ded85c40428d1cafa89f29702d, entries=150, sequenceid=286, filesize=12.0 K 2024-11-15T22:17:33,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/a4fd6617e989408ba3c64b4916f488bd as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a4fd6617e989408ba3c64b4916f488bd 2024-11-15T22:17:33,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709113068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a4fd6617e989408ba3c64b4916f488bd, entries=150, sequenceid=286, filesize=12.0 K 2024-11-15T22:17:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 00af3e32e9b483f8e15f8f4708045315 in 119ms, sequenceid=286, compaction requested=true 2024-11-15T22:17:33,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:33,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:33,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:33,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:33,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:33,093 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-15T22:17:33,093 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T22:17:33,094 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T22:17:33,094 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. because compaction request was cancelled 2024-11-15T22:17:33,094 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:A 2024-11-15T22:17:33,094 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:33,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:C, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:33,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:33,099 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:33,100 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/B is initiating minor compaction (all files) 2024-11-15T22:17:33,100 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/B in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:33,100 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/9f7ec0d00ce941b599fd33544dba68c9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/0a4f4410da394877afec08c995f595c8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/4be9e2ded85c40428d1cafa89f29702d] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=36.1 K 2024-11-15T22:17:33,100 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f7ec0d00ce941b599fd33544dba68c9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731709051082 2024-11-15T22:17:33,100 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a4f4410da394877afec08c995f595c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731709051770 2024-11-15T22:17:33,101 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 4be9e2ded85c40428d1cafa89f29702d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731709052944 2024-11-15T22:17:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:33,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-15T22:17:33,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:33,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:33,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:33,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:33,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:33,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:33,114 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:33,115 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/96e28e400ff5488a94f217d56447fa06 is 50, key is test_row_0/B:col10/1731709052972/Put/seqid=0 2024-11-15T22:17:33,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/4337f287d99f4bbfb956202102ad42fc is 50, key is test_row_0/A:col10/1731709053088/Put/seqid=0 2024-11-15T22:17:33,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742317_1493 (size=12609) 2024-11-15T22:17:33,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742318_1494 (size=14741) 2024-11-15T22:17:33,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/4337f287d99f4bbfb956202102ad42fc 2024-11-15T22:17:33,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709113125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,144 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/96e28e400ff5488a94f217d56447fa06 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/96e28e400ff5488a94f217d56447fa06 2024-11-15T22:17:33,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709113138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,154 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/B of 00af3e32e9b483f8e15f8f4708045315 into 96e28e400ff5488a94f217d56447fa06(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:33,154 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:33,154 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/B, priority=13, startTime=1731709053093; duration=0sec 2024-11-15T22:17:33,154 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:33,154 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:B 2024-11-15T22:17:33,154 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:33,156 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:33,156 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/C is initiating minor compaction (all files) 2024-11-15T22:17:33,156 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/C in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:33,157 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ffb964f0814846a188e23c5abce985d7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ce7fde65df4446e3a3482a7c47d1b225, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a4fd6617e989408ba3c64b4916f488bd] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=36.1 K 2024-11-15T22:17:33,157 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting ffb964f0814846a188e23c5abce985d7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731709051082 2024-11-15T22:17:33,158 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting ce7fde65df4446e3a3482a7c47d1b225, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731709051770 2024-11-15T22:17:33,159 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting a4fd6617e989408ba3c64b4916f488bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731709052944 2024-11-15T22:17:33,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/457fb55a9c584a7ca39c3d8acbefe693 is 50, key is test_row_0/B:col10/1731709053088/Put/seqid=0 2024-11-15T22:17:33,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709113140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709113142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,176 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#C#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:33,177 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/dd1d83b6ab5042cfafd3a0d01b33d874 is 50, key is test_row_0/C:col10/1731709052972/Put/seqid=0 2024-11-15T22:17:33,184 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/a334533295db407ca426062c2e31a52a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a334533295db407ca426062c2e31a52a 2024-11-15T22:17:33,190 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/A of 00af3e32e9b483f8e15f8f4708045315 into a334533295db407ca426062c2e31a52a(size=12.1 K), total size for store is 43.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:33,190 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:33,190 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/A, priority=13, startTime=1731709052740; duration=0sec 2024-11-15T22:17:33,190 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:33,190 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:A 2024-11-15T22:17:33,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742319_1495 (size=12301) 2024-11-15T22:17:33,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/457fb55a9c584a7ca39c3d8acbefe693 2024-11-15T22:17:33,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742320_1496 (size=12609) 2024-11-15T22:17:33,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709113201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,218 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/dd1d83b6ab5042cfafd3a0d01b33d874 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/dd1d83b6ab5042cfafd3a0d01b33d874 2024-11-15T22:17:33,224 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/C of 00af3e32e9b483f8e15f8f4708045315 into dd1d83b6ab5042cfafd3a0d01b33d874(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:33,224 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:33,224 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/C, priority=13, startTime=1731709053093; duration=0sec 2024-11-15T22:17:33,224 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:33,224 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:C 2024-11-15T22:17:33,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/947f158d5d95482c9a8b07f42455bb75 is 50, key is test_row_0/C:col10/1731709053088/Put/seqid=0 2024-11-15T22:17:33,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709113252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709113266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709113272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709113285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742321_1497 (size=12301) 2024-11-15T22:17:33,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709113421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709113491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709113501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709113501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709113501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/947f158d5d95482c9a8b07f42455bb75 2024-11-15T22:17:33,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/4337f287d99f4bbfb956202102ad42fc as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/4337f287d99f4bbfb956202102ad42fc 2024-11-15T22:17:33,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/4337f287d99f4bbfb956202102ad42fc, entries=200, sequenceid=322, filesize=14.4 K 2024-11-15T22:17:33,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/457fb55a9c584a7ca39c3d8acbefe693 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/457fb55a9c584a7ca39c3d8acbefe693 2024-11-15T22:17:33,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/457fb55a9c584a7ca39c3d8acbefe693, entries=150, sequenceid=322, filesize=12.0 K 2024-11-15T22:17:33,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/947f158d5d95482c9a8b07f42455bb75 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/947f158d5d95482c9a8b07f42455bb75 2024-11-15T22:17:33,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/947f158d5d95482c9a8b07f42455bb75, entries=150, sequenceid=322, filesize=12.0 K 2024-11-15T22:17:33,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 00af3e32e9b483f8e15f8f4708045315 in 617ms, sequenceid=322, compaction requested=true 2024-11-15T22:17:33,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:33,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:33,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:33,721 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:17:33,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:33,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:33,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:33,721 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T22:17:33,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:33,722 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T22:17:33,722 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T22:17:33,722 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. because compaction request was cancelled 2024-11-15T22:17:33,722 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:B 2024-11-15T22:17:33,722 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T22:17:33,723 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59020 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:17:33,723 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/A is initiating minor compaction (all files) 2024-11-15T22:17:33,723 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/A in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:33,723 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a334533295db407ca426062c2e31a52a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/4bc9636ec1ed4e9f92fd3e21efadc863, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/67556d93c7784746b59f586ed098fa51, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/4337f287d99f4bbfb956202102ad42fc] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=57.6 K 2024-11-15T22:17:33,723 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T22:17:33,723 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T22:17:33,723 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. because compaction request was cancelled 2024-11-15T22:17:33,723 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:C 2024-11-15T22:17:33,724 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting a334533295db407ca426062c2e31a52a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731709051082 2024-11-15T22:17:33,724 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bc9636ec1ed4e9f92fd3e21efadc863, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731709051770 2024-11-15T22:17:33,725 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67556d93c7784746b59f586ed098fa51, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731709052944 2024-11-15T22:17:33,725 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4337f287d99f4bbfb956202102ad42fc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731709053056 2024-11-15T22:17:33,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:33,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T22:17:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:33,757 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#A#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:33,757 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/8fab8ebcaffe4edba737d1c9b020e195 is 50, key is test_row_0/A:col10/1731709053088/Put/seqid=0 2024-11-15T22:17:33,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/9b27a2bcec17442aafa2eeaec13f11a0 is 50, key is test_row_0/A:col10/1731709053111/Put/seqid=0 2024-11-15T22:17:33,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742322_1498 (size=12643) 2024-11-15T22:17:33,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742323_1499 (size=14741) 2024-11-15T22:17:33,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/9b27a2bcec17442aafa2eeaec13f11a0 2024-11-15T22:17:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T22:17:33,832 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:33,833 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:17:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-15T22:17:33,835 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:17:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T22:17:33,836 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:17:33,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:17:33,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/cef37c5023c8466eaba9d73da7d8e971 is 50, key is test_row_0/B:col10/1731709053111/Put/seqid=0 2024-11-15T22:17:33,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731709053837, max timestamp=1731709053863Num uniques 859; 2024-11-15T22:17:33,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731709053837, max timestamp=1731709053863Num uniques 859; 2024-11-15T22:17:33,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T22:17:33,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T22:17:33,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731709053837, max timestamp=1731709053863Num uniques 859; 2024-11-15T22:17:33,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T22:17:33,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742324_1500 (size=12301) 2024-11-15T22:17:33,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709113898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709113910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709113911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709113913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709113915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T22:17:33,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T22:17:33,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:33,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:33,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:33,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:33,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:34,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709114022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709114022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:34,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709114041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:34,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709114041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:34,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709114041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:34,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T22:17:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:34,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:34,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:34,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T22:17:34,210 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/8fab8ebcaffe4edba737d1c9b020e195 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/8fab8ebcaffe4edba737d1c9b020e195 2024-11-15T22:17:34,214 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/A of 00af3e32e9b483f8e15f8f4708045315 into 8fab8ebcaffe4edba737d1c9b020e195(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:34,214 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:34,214 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/A, priority=12, startTime=1731709053721; duration=0sec 2024-11-15T22:17:34,214 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:34,214 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:A 2024-11-15T22:17:34,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709114242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:34,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:34,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709114242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:34,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:34,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:34,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:34,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709114261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:34,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709114261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:34,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709114261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:34,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/cef37c5023c8466eaba9d73da7d8e971 2024-11-15T22:17:34,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/ed0af1a583bc4edba080c071257cdd51 is 50, key is test_row_0/C:col10/1731709053111/Put/seqid=0 2024-11-15T22:17:34,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742325_1501 (size=12301) 2024-11-15T22:17:34,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T22:17:34,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:34,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:34,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:34,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:34,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:34,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T22:17:34,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:34,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:34,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:34,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:34,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T22:17:34,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709114553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:34,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709114553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:34,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709114571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:34,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709114581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:34,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709114581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:34,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T22:17:34,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:34,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:34,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:34,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:34,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:34,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/ed0af1a583bc4edba080c071257cdd51 2024-11-15T22:17:34,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/9b27a2bcec17442aafa2eeaec13f11a0 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/9b27a2bcec17442aafa2eeaec13f11a0 2024-11-15T22:17:34,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/9b27a2bcec17442aafa2eeaec13f11a0, entries=200, sequenceid=336, filesize=14.4 K 2024-11-15T22:17:34,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/cef37c5023c8466eaba9d73da7d8e971 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/cef37c5023c8466eaba9d73da7d8e971 2024-11-15T22:17:34,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/cef37c5023c8466eaba9d73da7d8e971, entries=150, sequenceid=336, filesize=12.0 K 2024-11-15T22:17:34,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/ed0af1a583bc4edba080c071257cdd51 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ed0af1a583bc4edba080c071257cdd51 2024-11-15T22:17:34,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ed0af1a583bc4edba080c071257cdd51, entries=150, sequenceid=336, filesize=12.0 K 2024-11-15T22:17:34,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 00af3e32e9b483f8e15f8f4708045315 in 966ms, sequenceid=336, compaction requested=true 2024-11-15T22:17:34,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:34,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:34,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:34,712 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T22:17:34,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:34,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:34,712 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:34,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:34,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:34,713 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T22:17:34,713 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T22:17:34,713 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. because compaction request was cancelled 2024-11-15T22:17:34,713 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:A 2024-11-15T22:17:34,713 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:34,716 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:34,716 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/B is initiating minor compaction (all files) 2024-11-15T22:17:34,716 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/B in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:34,716 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/96e28e400ff5488a94f217d56447fa06, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/457fb55a9c584a7ca39c3d8acbefe693, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/cef37c5023c8466eaba9d73da7d8e971] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=36.3 K 2024-11-15T22:17:34,717 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 96e28e400ff5488a94f217d56447fa06, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731709052944 2024-11-15T22:17:34,717 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 457fb55a9c584a7ca39c3d8acbefe693, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731709053056 2024-11-15T22:17:34,717 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting cef37c5023c8466eaba9d73da7d8e971, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731709053111 2024-11-15T22:17:34,718 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:34,718 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/C is initiating minor compaction (all files) 2024-11-15T22:17:34,718 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/C in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:34,718 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/dd1d83b6ab5042cfafd3a0d01b33d874, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/947f158d5d95482c9a8b07f42455bb75, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ed0af1a583bc4edba080c071257cdd51] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=36.3 K 2024-11-15T22:17:34,719 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd1d83b6ab5042cfafd3a0d01b33d874, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731709052944 2024-11-15T22:17:34,719 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 947f158d5d95482c9a8b07f42455bb75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731709053056 2024-11-15T22:17:34,720 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed0af1a583bc4edba080c071257cdd51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731709053111 2024-11-15T22:17:34,730 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:34,731 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/c94fb6b970da4c9daaed734f8110e431 is 50, key is test_row_0/B:col10/1731709053111/Put/seqid=0 2024-11-15T22:17:34,738 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#C#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:34,739 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/d26a30c9c6cf479db9a4840680867489 is 50, key is test_row_0/C:col10/1731709053111/Put/seqid=0 2024-11-15T22:17:34,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742326_1502 (size=12711) 2024-11-15T22:17:34,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T22:17:34,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:34,761 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-15T22:17:34,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:34,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:34,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:34,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:34,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:34,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:34,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/6e0cf3969376405baf9a4604e75f42d9 is 50, key is test_row_0/A:col10/1731709053911/Put/seqid=0 2024-11-15T22:17:34,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742327_1503 (size=12711) 2024-11-15T22:17:34,776 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/c94fb6b970da4c9daaed734f8110e431 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c94fb6b970da4c9daaed734f8110e431 2024-11-15T22:17:34,785 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/B of 00af3e32e9b483f8e15f8f4708045315 into c94fb6b970da4c9daaed734f8110e431(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:34,785 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:34,785 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/B, priority=13, startTime=1731709054712; duration=0sec 2024-11-15T22:17:34,785 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:34,785 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:B 2024-11-15T22:17:34,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742328_1504 (size=12301) 2024-11-15T22:17:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T22:17:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:35,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:35,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709115101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709115101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709115102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709115103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709115115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,181 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/d26a30c9c6cf479db9a4840680867489 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/d26a30c9c6cf479db9a4840680867489 2024-11-15T22:17:35,191 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/C of 00af3e32e9b483f8e15f8f4708045315 into d26a30c9c6cf479db9a4840680867489(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:35,191 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/6e0cf3969376405baf9a4604e75f42d9 2024-11-15T22:17:35,191 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:35,191 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/C, priority=13, startTime=1731709054712; duration=0sec 2024-11-15T22:17:35,191 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:35,191 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:C 2024-11-15T22:17:35,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/5ac5044d79de44e7a7c04129c94b4933 is 50, key is test_row_0/B:col10/1731709053911/Put/seqid=0 2024-11-15T22:17:35,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709115221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742329_1505 (size=12301) 2024-11-15T22:17:35,233 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/5ac5044d79de44e7a7c04129c94b4933 2024-11-15T22:17:35,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709115232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709115232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/c1f587620668463ca3a29c904216703e is 50, key is test_row_0/C:col10/1731709053911/Put/seqid=0 2024-11-15T22:17:35,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742330_1506 (size=12301) 2024-11-15T22:17:35,291 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/c1f587620668463ca3a29c904216703e 2024-11-15T22:17:35,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/6e0cf3969376405baf9a4604e75f42d9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/6e0cf3969376405baf9a4604e75f42d9 2024-11-15T22:17:35,311 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/6e0cf3969376405baf9a4604e75f42d9, entries=150, sequenceid=371, filesize=12.0 K 2024-11-15T22:17:35,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/5ac5044d79de44e7a7c04129c94b4933 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5ac5044d79de44e7a7c04129c94b4933 2024-11-15T22:17:35,319 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5ac5044d79de44e7a7c04129c94b4933, entries=150, sequenceid=371, filesize=12.0 K 2024-11-15T22:17:35,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/c1f587620668463ca3a29c904216703e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/c1f587620668463ca3a29c904216703e 2024-11-15T22:17:35,329 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/c1f587620668463ca3a29c904216703e, entries=150, sequenceid=371, filesize=12.0 K 2024-11-15T22:17:35,330 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 00af3e32e9b483f8e15f8f4708045315 in 569ms, sequenceid=371, compaction requested=true 2024-11-15T22:17:35,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:35,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:35,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-15T22:17:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-15T22:17:35,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-15T22:17:35,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5010 sec 2024-11-15T22:17:35,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.5070 sec 2024-11-15T22:17:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:35,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T22:17:35,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:35,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:35,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:35,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:35,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:35,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:35,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/76008348b4c44df5869cc8954fb13252 is 50, key is test_row_0/A:col10/1731709055435/Put/seqid=0 2024-11-15T22:17:35,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742331_1507 (size=14741) 2024-11-15T22:17:35,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/76008348b4c44df5869cc8954fb13252 2024-11-15T22:17:35,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/e809fbe2ffa241959257592fcc8e0740 is 50, key is test_row_0/B:col10/1731709055435/Put/seqid=0 2024-11-15T22:17:35,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742332_1508 (size=12301) 2024-11-15T22:17:35,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/e809fbe2ffa241959257592fcc8e0740 2024-11-15T22:17:35,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/a1cd188798364399969a78d169f3313d is 50, key is test_row_0/C:col10/1731709055435/Put/seqid=0 2024-11-15T22:17:35,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709115559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709115559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742333_1509 (size=12301) 2024-11-15T22:17:35,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/a1cd188798364399969a78d169f3313d 2024-11-15T22:17:35,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709115572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/76008348b4c44df5869cc8954fb13252 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/76008348b4c44df5869cc8954fb13252 2024-11-15T22:17:35,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/76008348b4c44df5869cc8954fb13252, entries=200, sequenceid=385, filesize=14.4 K 2024-11-15T22:17:35,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/e809fbe2ffa241959257592fcc8e0740 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/e809fbe2ffa241959257592fcc8e0740 2024-11-15T22:17:35,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/e809fbe2ffa241959257592fcc8e0740, entries=150, sequenceid=385, filesize=12.0 K 2024-11-15T22:17:35,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/a1cd188798364399969a78d169f3313d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a1cd188798364399969a78d169f3313d 2024-11-15T22:17:35,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a1cd188798364399969a78d169f3313d, entries=150, sequenceid=385, filesize=12.0 K 2024-11-15T22:17:35,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 00af3e32e9b483f8e15f8f4708045315 in 173ms, sequenceid=385, compaction requested=true 2024-11-15T22:17:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:35,609 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:17:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:35,609 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:35,610 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:35,610 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/B is initiating minor compaction (all files) 2024-11-15T22:17:35,610 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54426 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:17:35,610 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/B in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:35,610 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/A is initiating minor compaction (all files) 2024-11-15T22:17:35,610 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/A in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:35,610 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c94fb6b970da4c9daaed734f8110e431, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5ac5044d79de44e7a7c04129c94b4933, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/e809fbe2ffa241959257592fcc8e0740] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=36.4 K 2024-11-15T22:17:35,610 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/8fab8ebcaffe4edba737d1c9b020e195, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/9b27a2bcec17442aafa2eeaec13f11a0, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/6e0cf3969376405baf9a4604e75f42d9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/76008348b4c44df5869cc8954fb13252] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=53.2 K 2024-11-15T22:17:35,610 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c94fb6b970da4c9daaed734f8110e431, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731709053111 2024-11-15T22:17:35,610 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fab8ebcaffe4edba737d1c9b020e195, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731709053056 2024-11-15T22:17:35,611 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ac5044d79de44e7a7c04129c94b4933, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731709053908 2024-11-15T22:17:35,611 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b27a2bcec17442aafa2eeaec13f11a0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731709053111 2024-11-15T22:17:35,611 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting e809fbe2ffa241959257592fcc8e0740, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731709055094 2024-11-15T22:17:35,611 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e0cf3969376405baf9a4604e75f42d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731709053908 2024-11-15T22:17:35,616 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76008348b4c44df5869cc8954fb13252, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731709055094 2024-11-15T22:17:35,625 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#B#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:35,625 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/aa4c1560a0e546e7bc78aa07204ac511 is 50, key is test_row_0/B:col10/1731709055435/Put/seqid=0 2024-11-15T22:17:35,627 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#A#compaction#435 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:35,627 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/b3ec4a07eab44f97916b52844c86a294 is 50, key is test_row_0/A:col10/1731709055435/Put/seqid=0 2024-11-15T22:17:35,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742334_1510 (size=12813) 2024-11-15T22:17:35,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742335_1511 (size=12779) 2024-11-15T22:17:35,639 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/aa4c1560a0e546e7bc78aa07204ac511 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/aa4c1560a0e546e7bc78aa07204ac511 2024-11-15T22:17:35,645 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/B of 00af3e32e9b483f8e15f8f4708045315 into aa4c1560a0e546e7bc78aa07204ac511(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:35,645 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:35,646 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/B, priority=13, startTime=1731709055609; duration=0sec 2024-11-15T22:17:35,646 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:35,646 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:B 2024-11-15T22:17:35,646 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:35,646 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:35,646 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/C is initiating minor compaction (all files) 2024-11-15T22:17:35,647 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/C in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:35,647 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/d26a30c9c6cf479db9a4840680867489, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/c1f587620668463ca3a29c904216703e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a1cd188798364399969a78d169f3313d] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=36.4 K 2024-11-15T22:17:35,647 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d26a30c9c6cf479db9a4840680867489, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731709053111 2024-11-15T22:17:35,647 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c1f587620668463ca3a29c904216703e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731709053908 2024-11-15T22:17:35,648 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting a1cd188798364399969a78d169f3313d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731709055094 2024-11-15T22:17:35,652 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#C#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:35,653 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/d253e782065a4f0a964fde24523fdef5 is 50, key is test_row_0/C:col10/1731709055435/Put/seqid=0 2024-11-15T22:17:35,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742336_1512 (size=12813) 2024-11-15T22:17:35,664 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/d253e782065a4f0a964fde24523fdef5 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/d253e782065a4f0a964fde24523fdef5 2024-11-15T22:17:35,670 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/C of 00af3e32e9b483f8e15f8f4708045315 into d253e782065a4f0a964fde24523fdef5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:35,670 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:35,670 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/C, priority=13, startTime=1731709055609; duration=0sec 2024-11-15T22:17:35,670 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:35,670 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:C 2024-11-15T22:17:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:35,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T22:17:35,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731709055546, max timestamp=1731709055685Num uniques 859; 2024-11-15T22:17:35,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:17:35,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731709055546, max timestamp=1731709055685Num uniques 859; 2024-11-15T22:17:35,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:17:35,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:35,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:35,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:35,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:35,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:35,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:17:35,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/f451ef7807334866b3f8b114771a9d01 is 50, key is test_row_0/A:col10/1731709055686/Put/seqid=0 2024-11-15T22:17:35,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742337_1513 (size=17181) 2024-11-15T22:17:35,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709115843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709115844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709115860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709115972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709115972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:35,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709115972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:35,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:35,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T22:17:35,981 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:35,982 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:17:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-15T22:17:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T22:17:35,983 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:17:35,984 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:17:35,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:17:36,040 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/b3ec4a07eab44f97916b52844c86a294 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/b3ec4a07eab44f97916b52844c86a294 2024-11-15T22:17:36,043 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/A of 00af3e32e9b483f8e15f8f4708045315 into b3ec4a07eab44f97916b52844c86a294(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:36,043 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:36,044 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/A, priority=12, startTime=1731709055609; duration=0sec 2024-11-15T22:17:36,044 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:36,044 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:A 2024-11-15T22:17:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T22:17:36,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/f451ef7807334866b3f8b114771a9d01 2024-11-15T22:17:36,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/74c3f9cdb7234ce58cc65d487d975807 is 50, key is test_row_0/B:col10/1731709055686/Put/seqid=0 2024-11-15T22:17:36,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742338_1514 (size=12301) 2024-11-15T22:17:36,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/74c3f9cdb7234ce58cc65d487d975807 2024-11-15T22:17:36,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/eac04a1a34ab44d297fe802833bf5ea7 is 50, key is test_row_0/C:col10/1731709055686/Put/seqid=0 2024-11-15T22:17:36,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742339_1515 (size=12301) 2024-11-15T22:17:36,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T22:17:36,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:36,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:36,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:36,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:36,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:36,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709116131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:36,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709116132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:36,139 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in 00af3e32e9b483f8e15f8f4708045315 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:17:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:36,140 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in 00af3e32e9b483f8e15f8f4708045315 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:17:36,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:36,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709116181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:36,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:36,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709116181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:36,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:36,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709116181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:36,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:36,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T22:17:36,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:36,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:36,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:36,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:36,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T22:17:36,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709116492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709116492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:36,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709116492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T22:17:36,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/eac04a1a34ab44d297fe802833bf5ea7 2024-11-15T22:17:36,630 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1314861710_22 at /127.0.0.1:52110 [Receiving block BP-14505792-172.17.0.3-1731708921465:blk_1073741830_1006] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 340ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/cluster_a6235007-a423-ea83-cf16-0fd92ac68712/data/data2/, blockId=1073741830, seqno=1585 2024-11-15T22:17:36,630 INFO [AsyncFSWAL-0-hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData-prefix:e611192d6313,34969,1731708925722 {}] wal.AbstractFSWAL(1368): Slow sync cost: 340 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38455,DS-fbaa7e2b-1311-447c-a7c5-ccdf11aa0038,DISK]] 2024-11-15T22:17:36,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/f451ef7807334866b3f8b114771a9d01 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f451ef7807334866b3f8b114771a9d01 2024-11-15T22:17:36,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f451ef7807334866b3f8b114771a9d01, entries=250, sequenceid=409, filesize=16.8 K 2024-11-15T22:17:36,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/74c3f9cdb7234ce58cc65d487d975807 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/74c3f9cdb7234ce58cc65d487d975807 2024-11-15T22:17:36,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/74c3f9cdb7234ce58cc65d487d975807, entries=150, sequenceid=409, filesize=12.0 K 2024-11-15T22:17:36,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/eac04a1a34ab44d297fe802833bf5ea7 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/eac04a1a34ab44d297fe802833bf5ea7 2024-11-15T22:17:36,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/eac04a1a34ab44d297fe802833bf5ea7, entries=150, sequenceid=409, filesize=12.0 K 2024-11-15T22:17:36,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 00af3e32e9b483f8e15f8f4708045315 in 1016ms, sequenceid=409, compaction requested=false 2024-11-15T22:17:36,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:36,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T22:17:36,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:36,782 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T22:17:36,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:36,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:36,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:36,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:36,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:36,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/5c1382a598d64eb28bac7ca673419c79 is 50, key is test_row_0/A:col10/1731709055733/Put/seqid=0 2024-11-15T22:17:36,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742340_1516 (size=12301) 2024-11-15T22:17:37,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:37,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:37,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:37,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709117028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:37,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:37,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:37,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709117030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709117032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:37,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T22:17:37,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709117141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:37,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709117151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709117151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:37,294 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/5c1382a598d64eb28bac7ca673419c79 2024-11-15T22:17:37,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/005e18223a614e93bd0ee2656a406e06 is 50, key is test_row_0/B:col10/1731709055733/Put/seqid=0 2024-11-15T22:17:37,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742341_1517 (size=12301) 2024-11-15T22:17:37,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709117353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:37,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709117372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:37,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:37,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709117372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:37,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709117671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:37,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709117681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:37,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709117691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:37,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:37,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:37,721 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/005e18223a614e93bd0ee2656a406e06 2024-11-15T22:17:37,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/0b0b89715a1d4fb58f4824d7e1c36b18 is 50, key is test_row_0/C:col10/1731709055733/Put/seqid=0 2024-11-15T22:17:37,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742342_1518 (size=12301) 2024-11-15T22:17:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T22:17:38,133 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/0b0b89715a1d4fb58f4824d7e1c36b18 2024-11-15T22:17:38,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/5c1382a598d64eb28bac7ca673419c79 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/5c1382a598d64eb28bac7ca673419c79 2024-11-15T22:17:38,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709118164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:38,168 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in 00af3e32e9b483f8e15f8f4708045315 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:17:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:38,182 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/5c1382a598d64eb28bac7ca673419c79, entries=150, sequenceid=430, filesize=12.0 K 2024-11-15T22:17:38,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/005e18223a614e93bd0ee2656a406e06 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/005e18223a614e93bd0ee2656a406e06 2024-11-15T22:17:38,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709118171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:38,188 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in 00af3e32e9b483f8e15f8f4708045315 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4277 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:17:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:38,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:38,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709118197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:38,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709118204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:38,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709118212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:38,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:38,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:38,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:38,234 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/005e18223a614e93bd0ee2656a406e06, entries=150, sequenceid=430, filesize=12.0 K 2024-11-15T22:17:38,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/0b0b89715a1d4fb58f4824d7e1c36b18 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/0b0b89715a1d4fb58f4824d7e1c36b18 2024-11-15T22:17:38,247 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/0b0b89715a1d4fb58f4824d7e1c36b18, entries=150, sequenceid=430, filesize=12.0 K 2024-11-15T22:17:38,260 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 00af3e32e9b483f8e15f8f4708045315 in 1479ms, sequenceid=430, compaction requested=true 2024-11-15T22:17:38,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:38,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:38,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-15T22:17:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-15T22:17:38,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-15T22:17:38,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2790 sec 2024-11-15T22:17:38,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.2930 sec 2024-11-15T22:17:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:39,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T22:17:39,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:39,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:39,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:39,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:39,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:39,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:39,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/1824039e6d5d496d8f8622aa80d056c8 is 50, key is test_row_0/A:col10/1731709059237/Put/seqid=0 2024-11-15T22:17:39,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742343_1519 (size=19621) 2024-11-15T22:17:39,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709119395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:39,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709119425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:39,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709119429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:39,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709119532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:39,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709119576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:39,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709119576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:39,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:39,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:39,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/1824039e6d5d496d8f8622aa80d056c8 2024-11-15T22:17:39,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709119752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:39,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/f2eba6ecca4f41ea8456a9cf66d2ad70 is 50, key is test_row_0/B:col10/1731709059237/Put/seqid=0 2024-11-15T22:17:39,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709119804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742344_1520 (size=12301) 2024-11-15T22:17:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709119804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:39,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/f2eba6ecca4f41ea8456a9cf66d2ad70 2024-11-15T22:17:39,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/6cbf881a5d794d349cc020b469e442e8 is 50, key is test_row_0/C:col10/1731709059237/Put/seqid=0 2024-11-15T22:17:39,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742345_1521 (size=12301) 2024-11-15T22:17:40,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709120072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:40,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:40,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709120136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:40,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709120136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T22:17:40,165 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:40,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:17:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-15T22:17:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T22:17:40,176 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:17:40,197 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:17:40,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:17:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T22:17:40,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/6cbf881a5d794d349cc020b469e442e8 2024-11-15T22:17:40,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T22:17:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:40,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:40,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/1824039e6d5d496d8f8622aa80d056c8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/1824039e6d5d496d8f8622aa80d056c8 2024-11-15T22:17:40,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/1824039e6d5d496d8f8622aa80d056c8, entries=300, sequenceid=445, filesize=19.2 K 2024-11-15T22:17:40,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/f2eba6ecca4f41ea8456a9cf66d2ad70 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/f2eba6ecca4f41ea8456a9cf66d2ad70 2024-11-15T22:17:40,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/f2eba6ecca4f41ea8456a9cf66d2ad70, entries=150, sequenceid=445, filesize=12.0 K 2024-11-15T22:17:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T22:17:40,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/6cbf881a5d794d349cc020b469e442e8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/6cbf881a5d794d349cc020b469e442e8 2024-11-15T22:17:40,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T22:17:40,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:40,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:40,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:40,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:40,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:40,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/6cbf881a5d794d349cc020b469e442e8, entries=150, sequenceid=445, filesize=12.0 K 2024-11-15T22:17:40,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 00af3e32e9b483f8e15f8f4708045315 in 1306ms, sequenceid=445, compaction requested=true 2024-11-15T22:17:40,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:40,546 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:17:40,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:40,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:40,546 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:17:40,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:40,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:40,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:40,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:40,568 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:17:40,568 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/B is initiating minor compaction (all files) 2024-11-15T22:17:40,569 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/B in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:40,569 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/aa4c1560a0e546e7bc78aa07204ac511, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/74c3f9cdb7234ce58cc65d487d975807, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/005e18223a614e93bd0ee2656a406e06, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/f2eba6ecca4f41ea8456a9cf66d2ad70] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=48.6 K 2024-11-15T22:17:40,569 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 61882 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:17:40,569 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/A is initiating minor compaction (all files) 2024-11-15T22:17:40,569 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/A in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:40,569 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/b3ec4a07eab44f97916b52844c86a294, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f451ef7807334866b3f8b114771a9d01, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/5c1382a598d64eb28bac7ca673419c79, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/1824039e6d5d496d8f8622aa80d056c8] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=60.4 K 2024-11-15T22:17:40,572 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting aa4c1560a0e546e7bc78aa07204ac511, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731709055094 2024-11-15T22:17:40,572 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3ec4a07eab44f97916b52844c86a294, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731709055094 2024-11-15T22:17:40,575 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 74c3f9cdb7234ce58cc65d487d975807, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731709055546 2024-11-15T22:17:40,578 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting f451ef7807334866b3f8b114771a9d01, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731709055546 2024-11-15T22:17:40,580 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c1382a598d64eb28bac7ca673419c79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731709055733 2024-11-15T22:17:40,580 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 005e18223a614e93bd0ee2656a406e06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731709055733 2024-11-15T22:17:40,587 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1824039e6d5d496d8f8622aa80d056c8, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731709057007 2024-11-15T22:17:40,587 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting f2eba6ecca4f41ea8456a9cf66d2ad70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731709057019 2024-11-15T22:17:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:40,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T22:17:40,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:40,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:40,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:40,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:40,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:40,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:40,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/a425307909df4d3fbf63685ac6a00c6b is 50, key is test_row_0/A:col10/1731709059421/Put/seqid=0 2024-11-15T22:17:40,683 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#B#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:40,684 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/c9712cad31434c8588d81fc8b84acb1e is 50, key is test_row_0/B:col10/1731709059237/Put/seqid=0 2024-11-15T22:17:40,687 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#A#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:40,688 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/bc3f20d9cb0d41d7b8777cd9cb96d940 is 50, key is test_row_0/A:col10/1731709059237/Put/seqid=0 2024-11-15T22:17:40,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T22:17:40,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:40,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:40,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:40,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:40,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:40,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742346_1522 (size=14741) 2024-11-15T22:17:40,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742348_1524 (size=12915) 2024-11-15T22:17:40,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742347_1523 (size=12949) 2024-11-15T22:17:40,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709120727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:40,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709120728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709120728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:40,767 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/c9712cad31434c8588d81fc8b84acb1e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c9712cad31434c8588d81fc8b84acb1e 2024-11-15T22:17:40,777 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/B of 00af3e32e9b483f8e15f8f4708045315 into c9712cad31434c8588d81fc8b84acb1e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:40,777 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:40,777 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/B, priority=12, startTime=1731709060546; duration=0sec 2024-11-15T22:17:40,778 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:40,778 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/bc3f20d9cb0d41d7b8777cd9cb96d940 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/bc3f20d9cb0d41d7b8777cd9cb96d940 2024-11-15T22:17:40,778 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:B 2024-11-15T22:17:40,778 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:17:40,779 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:17:40,779 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/C is initiating minor compaction (all files) 2024-11-15T22:17:40,779 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/C in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:40,779 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/d253e782065a4f0a964fde24523fdef5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/eac04a1a34ab44d297fe802833bf5ea7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/0b0b89715a1d4fb58f4824d7e1c36b18, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/6cbf881a5d794d349cc020b469e442e8] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=48.6 K 2024-11-15T22:17:40,780 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d253e782065a4f0a964fde24523fdef5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731709055094 2024-11-15T22:17:40,780 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting eac04a1a34ab44d297fe802833bf5ea7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731709055546 2024-11-15T22:17:40,780 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b0b89715a1d4fb58f4824d7e1c36b18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731709055733 2024-11-15T22:17:40,780 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cbf881a5d794d349cc020b469e442e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731709057019 2024-11-15T22:17:40,786 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/A of 00af3e32e9b483f8e15f8f4708045315 into bc3f20d9cb0d41d7b8777cd9cb96d940(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:40,786 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:40,786 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/A, priority=12, startTime=1731709060546; duration=0sec 2024-11-15T22:17:40,786 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:40,786 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:A 2024-11-15T22:17:40,795 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#C#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:40,796 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/1545a1fcced04f5f8e3db7f8d4762774 is 50, key is test_row_0/C:col10/1731709059237/Put/seqid=0 2024-11-15T22:17:40,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T22:17:40,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742349_1525 (size=12949) 2024-11-15T22:17:40,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T22:17:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:40,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:40,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:40,872 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/1545a1fcced04f5f8e3db7f8d4762774 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/1545a1fcced04f5f8e3db7f8d4762774 2024-11-15T22:17:40,885 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/C of 00af3e32e9b483f8e15f8f4708045315 into 1545a1fcced04f5f8e3db7f8d4762774(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:40,885 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:40,885 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/C, priority=12, startTime=1731709060547; duration=0sec 2024-11-15T22:17:40,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:40,885 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:40,885 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:C 2024-11-15T22:17:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709120862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709120864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:40,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709120863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:41,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T22:17:41,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:41,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:41,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709121092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:41,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:41,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709121092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:41,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/a425307909df4d3fbf63685ac6a00c6b 2024-11-15T22:17:41,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709121104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:41,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/5f08d250a08a4e318d09623b9fffb38a is 50, key is test_row_0/B:col10/1731709059421/Put/seqid=0 2024-11-15T22:17:41,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742350_1526 (size=12301) 2024-11-15T22:17:41,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/5f08d250a08a4e318d09623b9fffb38a 2024-11-15T22:17:41,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T22:17:41,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:41,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/3f360c8d1741407c931cd96f2aa3e44a is 50, key is test_row_0/C:col10/1731709059421/Put/seqid=0 2024-11-15T22:17:41,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742351_1527 (size=12301) 2024-11-15T22:17:41,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/3f360c8d1741407c931cd96f2aa3e44a 2024-11-15T22:17:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T22:17:41,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/a425307909df4d3fbf63685ac6a00c6b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a425307909df4d3fbf63685ac6a00c6b 2024-11-15T22:17:41,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a425307909df4d3fbf63685ac6a00c6b, entries=200, sequenceid=463, filesize=14.4 K 2024-11-15T22:17:41,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/5f08d250a08a4e318d09623b9fffb38a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5f08d250a08a4e318d09623b9fffb38a 2024-11-15T22:17:41,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5f08d250a08a4e318d09623b9fffb38a, entries=150, sequenceid=463, filesize=12.0 K 2024-11-15T22:17:41,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/3f360c8d1741407c931cd96f2aa3e44a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/3f360c8d1741407c931cd96f2aa3e44a 2024-11-15T22:17:41,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T22:17:41,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:41,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/3f360c8d1741407c931cd96f2aa3e44a, entries=150, sequenceid=463, filesize=12.0 K 2024-11-15T22:17:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 00af3e32e9b483f8e15f8f4708045315 in 741ms, sequenceid=463, compaction requested=false 2024-11-15T22:17:41,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:41,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T22:17:41,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:41,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:41,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:41,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:41,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:41,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:41,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/c160e4ab1d054f3683a1a56da93f6ca8 is 50, key is test_row_0/A:col10/1731709060724/Put/seqid=0 2024-11-15T22:17:41,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742352_1528 (size=14741) 2024-11-15T22:17:41,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/c160e4ab1d054f3683a1a56da93f6ca8 2024-11-15T22:17:41,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/cad91da4537042bf9c70f417bfb10efe is 50, key is test_row_0/B:col10/1731709060724/Put/seqid=0 2024-11-15T22:17:41,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742353_1529 (size=12301) 2024-11-15T22:17:41,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/cad91da4537042bf9c70f417bfb10efe 2024-11-15T22:17:41,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T22:17:41,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:41,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709121512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:41,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:41,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709121516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:41,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709121535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:41,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/e5f43d48d3604375ab436f7854998894 is 50, key is test_row_0/C:col10/1731709060724/Put/seqid=0 2024-11-15T22:17:41,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742354_1530 (size=12301) 2024-11-15T22:17:41,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/e5f43d48d3604375ab436f7854998894 2024-11-15T22:17:41,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/c160e4ab1d054f3683a1a56da93f6ca8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/c160e4ab1d054f3683a1a56da93f6ca8 2024-11-15T22:17:41,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/c160e4ab1d054f3683a1a56da93f6ca8, entries=200, sequenceid=481, filesize=14.4 K 2024-11-15T22:17:41,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/cad91da4537042bf9c70f417bfb10efe as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/cad91da4537042bf9c70f417bfb10efe 2024-11-15T22:17:41,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/cad91da4537042bf9c70f417bfb10efe, entries=150, sequenceid=481, filesize=12.0 K 2024-11-15T22:17:41,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/e5f43d48d3604375ab436f7854998894 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/e5f43d48d3604375ab436f7854998894 2024-11-15T22:17:41,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/e5f43d48d3604375ab436f7854998894, entries=150, sequenceid=481, filesize=12.0 K 2024-11-15T22:17:41,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 00af3e32e9b483f8e15f8f4708045315 in 177ms, sequenceid=481, compaction requested=true 2024-11-15T22:17:41,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:41,605 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:41,606 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42397 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:41,606 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/A is initiating minor compaction (all files) 2024-11-15T22:17:41,606 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/A in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,606 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/bc3f20d9cb0d41d7b8777cd9cb96d940, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a425307909df4d3fbf63685ac6a00c6b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/c160e4ab1d054f3683a1a56da93f6ca8] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=41.4 K 2024-11-15T22:17:41,607 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc3f20d9cb0d41d7b8777cd9cb96d940, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731709057019 2024-11-15T22:17:41,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:17:41,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:41,607 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting a425307909df4d3fbf63685ac6a00c6b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731709059381 2024-11-15T22:17:41,607 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:41,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:17:41,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:41,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00af3e32e9b483f8e15f8f4708045315:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:17:41,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:41,608 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting c160e4ab1d054f3683a1a56da93f6ca8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1731709060723 2024-11-15T22:17:41,609 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:41,609 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/B is initiating minor compaction (all files) 2024-11-15T22:17:41,609 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/B in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,609 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c9712cad31434c8588d81fc8b84acb1e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5f08d250a08a4e318d09623b9fffb38a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/cad91da4537042bf9c70f417bfb10efe] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=36.7 K 2024-11-15T22:17:41,610 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c9712cad31434c8588d81fc8b84acb1e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731709057019 2024-11-15T22:17:41,610 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f08d250a08a4e318d09623b9fffb38a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731709059407 2024-11-15T22:17:41,610 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting cad91da4537042bf9c70f417bfb10efe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1731709060724 2024-11-15T22:17:41,619 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#A#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:41,619 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/caead73866ce4313bd91572a3a1ad879 is 50, key is test_row_0/A:col10/1731709060724/Put/seqid=0 2024-11-15T22:17:41,626 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:41,627 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/520386171c21425eb5eef6232bcc1711 is 50, key is test_row_0/B:col10/1731709060724/Put/seqid=0 2024-11-15T22:17:41,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742355_1531 (size=13017) 2024-11-15T22:17:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:41,661 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/caead73866ce4313bd91572a3a1ad879 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/caead73866ce4313bd91572a3a1ad879 2024-11-15T22:17:41,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T22:17:41,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:41,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:41,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:41,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:41,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:41,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:41,675 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/A of 00af3e32e9b483f8e15f8f4708045315 into caead73866ce4313bd91572a3a1ad879(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:41,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/aba7df4ac9da433f940c47f92c80bc02 is 50, key is test_row_0/A:col10/1731709061503/Put/seqid=0 2024-11-15T22:17:41,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742356_1532 (size=13051) 2024-11-15T22:17:41,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T22:17:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,692 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:41,692 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/A, priority=13, startTime=1731709061605; duration=0sec 2024-11-15T22:17:41,693 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:17:41,693 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:A 2024-11-15T22:17:41,693 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:17:41,699 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:17:41,699 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): 00af3e32e9b483f8e15f8f4708045315/C is initiating minor compaction (all files) 2024-11-15T22:17:41,699 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00af3e32e9b483f8e15f8f4708045315/C in TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,699 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/1545a1fcced04f5f8e3db7f8d4762774, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/3f360c8d1741407c931cd96f2aa3e44a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/e5f43d48d3604375ab436f7854998894] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp, totalSize=36.7 K 2024-11-15T22:17:41,699 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1545a1fcced04f5f8e3db7f8d4762774, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731709057019 2024-11-15T22:17:41,699 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f360c8d1741407c931cd96f2aa3e44a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731709059407 2024-11-15T22:17:41,699 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5f43d48d3604375ab436f7854998894, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1731709060724 2024-11-15T22:17:41,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742357_1533 (size=14741) 2024-11-15T22:17:41,729 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00af3e32e9b483f8e15f8f4708045315#C#compaction#458 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:17:41,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/aba7df4ac9da433f940c47f92c80bc02 2024-11-15T22:17:41,731 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/7078a879409840a39fea79f1f13cbe95 is 50, key is test_row_0/C:col10/1731709060724/Put/seqid=0 2024-11-15T22:17:41,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709121720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:41,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709121728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:41,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:41,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/fccebae66d8344eb9e5f29d5b4161f8e is 50, key is test_row_0/B:col10/1731709061503/Put/seqid=0 2024-11-15T22:17:41,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709121731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:41,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742358_1534 (size=13051) 2024-11-15T22:17:41,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742359_1535 (size=12301) 2024-11-15T22:17:41,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T22:17:41,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:41,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709121841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:41,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709121853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:41,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709121861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:41,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T22:17:41,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:41,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:41,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:42,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37594 deadline: 1731709122062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:42,076 INFO [Thread-1970 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1970. 2024-11-15T22:17:42,076 DEBUG [Thread-1970 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:42,076 DEBUG [Thread-1970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,077 DEBUG [Thread-1970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,077 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:42,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37570 deadline: 1731709122071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:42,083 INFO [Thread-1968 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1968. 2024-11-15T22:17:42,083 DEBUG [Thread-1968 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:42,083 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,083 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37586 deadline: 1731709122084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:42,088 INFO [Thread-1966 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1966. 2024-11-15T22:17:42,088 DEBUG [Thread-1966 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:42,088 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:42,091 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,091 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:42,093 INFO [Thread-1972 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1972. 2024-11-15T22:17:42,093 DEBUG [Thread-1972 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:42,093 DEBUG [Thread-1972 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,093 DEBUG [Thread-1972 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,093 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:42,109 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/520386171c21425eb5eef6232bcc1711 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/520386171c21425eb5eef6232bcc1711 2024-11-15T22:17:42,111 INFO [Thread-1974 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1974. 2024-11-15T22:17:42,111 DEBUG [Thread-1974 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:42,111 DEBUG [Thread-1974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,111 DEBUG [Thread-1974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:42,127 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/B of 00af3e32e9b483f8e15f8f4708045315 into 520386171c21425eb5eef6232bcc1711(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:42,128 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:42,128 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/B, priority=13, startTime=1731709061607; duration=0sec 2024-11-15T22:17:42,128 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:42,128 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:B 2024-11-15T22:17:42,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T22:17:42,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:42,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:42,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:42,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:42,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:42,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37630 deadline: 1731709122192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:42,193 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in 00af3e32e9b483f8e15f8f4708045315 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:17:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:42,198 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/7078a879409840a39fea79f1f13cbe95 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/7078a879409840a39fea79f1f13cbe95 2024-11-15T22:17:42,206 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00af3e32e9b483f8e15f8f4708045315/C of 00af3e32e9b483f8e15f8f4708045315 into 7078a879409840a39fea79f1f13cbe95(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:17:42,206 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:42,207 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., storeName=00af3e32e9b483f8e15f8f4708045315/C, priority=13, startTime=1731709061607; duration=0sec 2024-11-15T22:17:42,207 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:17:42,207 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00af3e32e9b483f8e15f8f4708045315:C 2024-11-15T22:17:42,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37608 deadline: 1731709122211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:42,212 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in 00af3e32e9b483f8e15f8f4708045315 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8301 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:17:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 , the old value is region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., hostname=e611192d6313,38779,1731708927034, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:17:42,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/fccebae66d8344eb9e5f29d5b4161f8e 2024-11-15T22:17:42,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/2509d8c49c0346d3a6ccf76784849f02 is 50, key is test_row_0/C:col10/1731709061503/Put/seqid=0 2024-11-15T22:17:42,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742360_1536 (size=12301) 2024-11-15T22:17:42,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/2509d8c49c0346d3a6ccf76784849f02 2024-11-15T22:17:42,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/aba7df4ac9da433f940c47f92c80bc02 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/aba7df4ac9da433f940c47f92c80bc02 2024-11-15T22:17:42,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/aba7df4ac9da433f940c47f92c80bc02, entries=200, sequenceid=501, filesize=14.4 K 2024-11-15T22:17:42,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/fccebae66d8344eb9e5f29d5b4161f8e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/fccebae66d8344eb9e5f29d5b4161f8e 2024-11-15T22:17:42,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/fccebae66d8344eb9e5f29d5b4161f8e, entries=150, sequenceid=501, filesize=12.0 K 2024-11-15T22:17:42,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/2509d8c49c0346d3a6ccf76784849f02 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/2509d8c49c0346d3a6ccf76784849f02 2024-11-15T22:17:42,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/2509d8c49c0346d3a6ccf76784849f02, entries=150, sequenceid=501, filesize=12.0 K 2024-11-15T22:17:42,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 00af3e32e9b483f8e15f8f4708045315 in 636ms, sequenceid=501, compaction requested=false 2024-11-15T22:17:42,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T22:17:42,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T22:17:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:42,323 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T22:17:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:42,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/641cfe0fc55a4c9e906056e5069484e9 is 50, key is test_row_0/A:col10/1731709061723/Put/seqid=0 2024-11-15T22:17:42,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742361_1537 (size=12301) 2024-11-15T22:17:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:42,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. as already flushing 2024-11-15T22:17:42,383 INFO [Thread-1961 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1961. 2024-11-15T22:17:42,383 DEBUG [Thread-1961 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:42,383 DEBUG [Thread-1961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,383 DEBUG [Thread-1961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:42,393 INFO [Thread-1963 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1963. 2024-11-15T22:17:42,393 INFO [Thread-1955 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1955. 2024-11-15T22:17:42,393 DEBUG [Thread-1963 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:42,393 DEBUG [Thread-1955 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:42,393 DEBUG [Thread-1963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,393 DEBUG [Thread-1955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,393 DEBUG [Thread-1955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,393 DEBUG [Thread-1963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:42,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:42,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:42,733 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/641cfe0fc55a4c9e906056e5069484e9 2024-11-15T22:17:42,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/90b64ce1f8374a46b03044e285182918 is 50, key is test_row_0/B:col10/1731709061723/Put/seqid=0 2024-11-15T22:17:42,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742362_1538 (size=12301) 2024-11-15T22:17:42,782 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/90b64ce1f8374a46b03044e285182918 2024-11-15T22:17:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/149ad284c1ce420ea195849de789f50e is 50, key is test_row_0/C:col10/1731709061723/Put/seqid=0 2024-11-15T22:17:42,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742363_1539 (size=12301) 2024-11-15T22:17:42,818 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/149ad284c1ce420ea195849de789f50e 2024-11-15T22:17:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/641cfe0fc55a4c9e906056e5069484e9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/641cfe0fc55a4c9e906056e5069484e9 2024-11-15T22:17:42,854 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/641cfe0fc55a4c9e906056e5069484e9, entries=150, sequenceid=516, filesize=12.0 K 2024-11-15T22:17:42,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/90b64ce1f8374a46b03044e285182918 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/90b64ce1f8374a46b03044e285182918 2024-11-15T22:17:42,879 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/90b64ce1f8374a46b03044e285182918, entries=150, sequenceid=516, filesize=12.0 K 2024-11-15T22:17:42,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/149ad284c1ce420ea195849de789f50e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/149ad284c1ce420ea195849de789f50e 2024-11-15T22:17:42,885 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/149ad284c1ce420ea195849de789f50e, entries=150, sequenceid=516, filesize=12.0 K 2024-11-15T22:17:42,888 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for 00af3e32e9b483f8e15f8f4708045315 in 565ms, sequenceid=516, compaction requested=true 2024-11-15T22:17:42,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 00af3e32e9b483f8e15f8f4708045315: 2024-11-15T22:17:42,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:42,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-15T22:17:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-15T22:17:42,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-15T22:17:42,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7040 sec 2024-11-15T22:17:42,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.7380 sec 2024-11-15T22:17:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T22:17:44,332 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:52,293 INFO [Thread-1957 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1957. 2024-11-15T22:17:52,293 DEBUG [Thread-1957 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:52,293 DEBUG [Thread-1957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:52,293 DEBUG [Thread-1957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:52,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:52,313 INFO [Thread-1959 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1959. 2024-11-15T22:17:52,313 DEBUG [Thread-1959 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:17:52,313 DEBUG [Thread-1959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:52,314 DEBUG [Thread-1959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:52,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 863 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2589 rows 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 857 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2569 rows 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 860 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2580 rows 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 866 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2593 rows 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 863 2024-11-15T22:17:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2589 rows 2024-11-15T22:17:52,314 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:17:52,314 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:17:52,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:52,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:52,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:17:52,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T22:17:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T22:17:52,319 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731709072319"}]},"ts":"1731709072319"} 2024-11-15T22:17:52,321 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T22:17:52,321 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T22:17:52,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:17:52,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00af3e32e9b483f8e15f8f4708045315, UNASSIGN}] 2024-11-15T22:17:52,325 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00af3e32e9b483f8e15f8f4708045315, UNASSIGN 2024-11-15T22:17:52,326 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=00af3e32e9b483f8e15f8f4708045315, regionState=CLOSING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:17:52,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00af3e32e9b483f8e15f8f4708045315, UNASSIGN because future has completed 2024-11-15T22:17:52,329 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:17:52,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:17:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T22:17:52,481 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(122): Close 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:52,481 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:17:52,481 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1722): Closing 00af3e32e9b483f8e15f8f4708045315, disabling compactions & flushes 2024-11-15T22:17:52,481 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:52,481 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:52,481 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. after waiting 0 ms 2024-11-15T22:17:52,481 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:52,481 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(2902): Flushing 00af3e32e9b483f8e15f8f4708045315 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T22:17:52,482 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=A 2024-11-15T22:17:52,482 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:52,482 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=B 2024-11-15T22:17:52,482 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:52,482 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00af3e32e9b483f8e15f8f4708045315, store=C 2024-11-15T22:17:52,482 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:52,485 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/b98281259a404c0ca16df8fb20aae01c is 50, key is test_row_0/A:col10/1731709062381/Put/seqid=0 2024-11-15T22:17:52,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742364_1540 (size=12301) 2024-11-15T22:17:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T22:17:52,894 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/b98281259a404c0ca16df8fb20aae01c 2024-11-15T22:17:52,904 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/f2c1c9d26d88467e8593ba87150dcd2e is 50, key is test_row_0/B:col10/1731709062381/Put/seqid=0 2024-11-15T22:17:52,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742365_1541 (size=12301) 2024-11-15T22:17:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T22:17:53,310 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/f2c1c9d26d88467e8593ba87150dcd2e 2024-11-15T22:17:53,316 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/41a8e57cdf404e6294708c2c64a5a236 is 50, key is test_row_0/C:col10/1731709062381/Put/seqid=0 2024-11-15T22:17:53,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742366_1542 (size=12301) 2024-11-15T22:17:53,323 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/41a8e57cdf404e6294708c2c64a5a236 2024-11-15T22:17:53,329 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/A/b98281259a404c0ca16df8fb20aae01c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/b98281259a404c0ca16df8fb20aae01c 2024-11-15T22:17:53,333 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/b98281259a404c0ca16df8fb20aae01c, entries=150, sequenceid=524, filesize=12.0 K 2024-11-15T22:17:53,333 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/B/f2c1c9d26d88467e8593ba87150dcd2e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/f2c1c9d26d88467e8593ba87150dcd2e 2024-11-15T22:17:53,338 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/f2c1c9d26d88467e8593ba87150dcd2e, entries=150, sequenceid=524, filesize=12.0 K 2024-11-15T22:17:53,338 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/.tmp/C/41a8e57cdf404e6294708c2c64a5a236 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/41a8e57cdf404e6294708c2c64a5a236 2024-11-15T22:17:53,342 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/41a8e57cdf404e6294708c2c64a5a236, entries=150, sequenceid=524, filesize=12.0 K 2024-11-15T22:17:53,343 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 00af3e32e9b483f8e15f8f4708045315 in 862ms, sequenceid=524, compaction requested=true 2024-11-15T22:17:53,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a24f4270692c41da8e912a56f56e69e2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f2bafdc6fa8640bab286936226459d50, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/03df7b08a49b49c5a5b83e2bcdd49493, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/e2c861c97e6d405f8721dc5816d03552, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/771d581b9d964062bee4e0cc2414d40b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/854d88e0bd204d479c6c7ad0612b7f35, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a334533295db407ca426062c2e31a52a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/4bc9636ec1ed4e9f92fd3e21efadc863, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/67556d93c7784746b59f586ed098fa51, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/4337f287d99f4bbfb956202102ad42fc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/8fab8ebcaffe4edba737d1c9b020e195, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/9b27a2bcec17442aafa2eeaec13f11a0, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/6e0cf3969376405baf9a4604e75f42d9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/76008348b4c44df5869cc8954fb13252, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/b3ec4a07eab44f97916b52844c86a294, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f451ef7807334866b3f8b114771a9d01, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/5c1382a598d64eb28bac7ca673419c79, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/1824039e6d5d496d8f8622aa80d056c8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/bc3f20d9cb0d41d7b8777cd9cb96d940, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a425307909df4d3fbf63685ac6a00c6b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/c160e4ab1d054f3683a1a56da93f6ca8] to archive 2024-11-15T22:17:53,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:17:53,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a24f4270692c41da8e912a56f56e69e2 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a24f4270692c41da8e912a56f56e69e2 2024-11-15T22:17:53,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f2bafdc6fa8640bab286936226459d50 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f2bafdc6fa8640bab286936226459d50 2024-11-15T22:17:53,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/03df7b08a49b49c5a5b83e2bcdd49493 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/03df7b08a49b49c5a5b83e2bcdd49493 2024-11-15T22:17:53,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/e2c861c97e6d405f8721dc5816d03552 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/e2c861c97e6d405f8721dc5816d03552 2024-11-15T22:17:53,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/771d581b9d964062bee4e0cc2414d40b to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/771d581b9d964062bee4e0cc2414d40b 2024-11-15T22:17:53,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/854d88e0bd204d479c6c7ad0612b7f35 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/854d88e0bd204d479c6c7ad0612b7f35 2024-11-15T22:17:53,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a334533295db407ca426062c2e31a52a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a334533295db407ca426062c2e31a52a 2024-11-15T22:17:53,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/4bc9636ec1ed4e9f92fd3e21efadc863 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/4bc9636ec1ed4e9f92fd3e21efadc863 2024-11-15T22:17:53,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/67556d93c7784746b59f586ed098fa51 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/67556d93c7784746b59f586ed098fa51 2024-11-15T22:17:53,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/4337f287d99f4bbfb956202102ad42fc to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/4337f287d99f4bbfb956202102ad42fc 2024-11-15T22:17:53,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/8fab8ebcaffe4edba737d1c9b020e195 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/8fab8ebcaffe4edba737d1c9b020e195 2024-11-15T22:17:53,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/9b27a2bcec17442aafa2eeaec13f11a0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/9b27a2bcec17442aafa2eeaec13f11a0 2024-11-15T22:17:53,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/6e0cf3969376405baf9a4604e75f42d9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/6e0cf3969376405baf9a4604e75f42d9 2024-11-15T22:17:53,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/76008348b4c44df5869cc8954fb13252 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/76008348b4c44df5869cc8954fb13252 2024-11-15T22:17:53,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/b3ec4a07eab44f97916b52844c86a294 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/b3ec4a07eab44f97916b52844c86a294 2024-11-15T22:17:53,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f451ef7807334866b3f8b114771a9d01 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/f451ef7807334866b3f8b114771a9d01 2024-11-15T22:17:53,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/5c1382a598d64eb28bac7ca673419c79 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/5c1382a598d64eb28bac7ca673419c79 2024-11-15T22:17:53,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/1824039e6d5d496d8f8622aa80d056c8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/1824039e6d5d496d8f8622aa80d056c8 2024-11-15T22:17:53,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/bc3f20d9cb0d41d7b8777cd9cb96d940 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/bc3f20d9cb0d41d7b8777cd9cb96d940 2024-11-15T22:17:53,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a425307909df4d3fbf63685ac6a00c6b to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/a425307909df4d3fbf63685ac6a00c6b 2024-11-15T22:17:53,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/c160e4ab1d054f3683a1a56da93f6ca8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/c160e4ab1d054f3683a1a56da93f6ca8 2024-11-15T22:17:53,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/1011fd5361434ef597a4d004288e5c67, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5fd6395e0c374d7fb709661caf038e10, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/365fd7753ce04100b46d9522a03838e5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/045591a206684de2bdd3b17e737f43ed, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5543934eede446118d04d85037bc8755, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/9f7ec0d00ce941b599fd33544dba68c9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/e839b8797a0d4a758e97cd8c33c73674, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/0a4f4410da394877afec08c995f595c8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/96e28e400ff5488a94f217d56447fa06, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/4be9e2ded85c40428d1cafa89f29702d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/457fb55a9c584a7ca39c3d8acbefe693, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c94fb6b970da4c9daaed734f8110e431, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/cef37c5023c8466eaba9d73da7d8e971, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5ac5044d79de44e7a7c04129c94b4933, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/aa4c1560a0e546e7bc78aa07204ac511, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/e809fbe2ffa241959257592fcc8e0740, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/74c3f9cdb7234ce58cc65d487d975807, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/005e18223a614e93bd0ee2656a406e06, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c9712cad31434c8588d81fc8b84acb1e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/f2eba6ecca4f41ea8456a9cf66d2ad70, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5f08d250a08a4e318d09623b9fffb38a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/cad91da4537042bf9c70f417bfb10efe] to archive 2024-11-15T22:17:53,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:17:53,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/1011fd5361434ef597a4d004288e5c67 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/1011fd5361434ef597a4d004288e5c67 2024-11-15T22:17:53,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5fd6395e0c374d7fb709661caf038e10 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5fd6395e0c374d7fb709661caf038e10 2024-11-15T22:17:53,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/365fd7753ce04100b46d9522a03838e5 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/365fd7753ce04100b46d9522a03838e5 2024-11-15T22:17:53,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/045591a206684de2bdd3b17e737f43ed to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/045591a206684de2bdd3b17e737f43ed 2024-11-15T22:17:53,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5543934eede446118d04d85037bc8755 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5543934eede446118d04d85037bc8755 2024-11-15T22:17:53,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/9f7ec0d00ce941b599fd33544dba68c9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/9f7ec0d00ce941b599fd33544dba68c9 2024-11-15T22:17:53,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/e839b8797a0d4a758e97cd8c33c73674 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/e839b8797a0d4a758e97cd8c33c73674 2024-11-15T22:17:53,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/0a4f4410da394877afec08c995f595c8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/0a4f4410da394877afec08c995f595c8 2024-11-15T22:17:53,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/96e28e400ff5488a94f217d56447fa06 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/96e28e400ff5488a94f217d56447fa06 2024-11-15T22:17:53,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/4be9e2ded85c40428d1cafa89f29702d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/4be9e2ded85c40428d1cafa89f29702d 2024-11-15T22:17:53,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/457fb55a9c584a7ca39c3d8acbefe693 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/457fb55a9c584a7ca39c3d8acbefe693 2024-11-15T22:17:53,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c94fb6b970da4c9daaed734f8110e431 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c94fb6b970da4c9daaed734f8110e431 2024-11-15T22:17:53,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/cef37c5023c8466eaba9d73da7d8e971 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/cef37c5023c8466eaba9d73da7d8e971 2024-11-15T22:17:53,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5ac5044d79de44e7a7c04129c94b4933 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5ac5044d79de44e7a7c04129c94b4933 2024-11-15T22:17:53,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/aa4c1560a0e546e7bc78aa07204ac511 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/aa4c1560a0e546e7bc78aa07204ac511 2024-11-15T22:17:53,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/e809fbe2ffa241959257592fcc8e0740 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/e809fbe2ffa241959257592fcc8e0740 2024-11-15T22:17:53,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/74c3f9cdb7234ce58cc65d487d975807 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/74c3f9cdb7234ce58cc65d487d975807 2024-11-15T22:17:53,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/005e18223a614e93bd0ee2656a406e06 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/005e18223a614e93bd0ee2656a406e06 2024-11-15T22:17:53,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c9712cad31434c8588d81fc8b84acb1e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/c9712cad31434c8588d81fc8b84acb1e 2024-11-15T22:17:53,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/f2eba6ecca4f41ea8456a9cf66d2ad70 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/f2eba6ecca4f41ea8456a9cf66d2ad70 2024-11-15T22:17:53,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5f08d250a08a4e318d09623b9fffb38a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/5f08d250a08a4e318d09623b9fffb38a 2024-11-15T22:17:53,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/cad91da4537042bf9c70f417bfb10efe to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/cad91da4537042bf9c70f417bfb10efe 2024-11-15T22:17:53,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/5f159bf16a7741a480568d465cee4735, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/81b0c49a07fd434e88cd09caae26384e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/7b9304b1859844fcad93ece6375d9bb2, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/9ab86da33aad4d77816864f68b0e4500, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/3c480bedb22d432fa387de393c9a6708, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ffb964f0814846a188e23c5abce985d7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/c18f8810557a48cb819f4d62c7ddade6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ce7fde65df4446e3a3482a7c47d1b225, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/dd1d83b6ab5042cfafd3a0d01b33d874, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a4fd6617e989408ba3c64b4916f488bd, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/947f158d5d95482c9a8b07f42455bb75, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/d26a30c9c6cf479db9a4840680867489, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ed0af1a583bc4edba080c071257cdd51, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/c1f587620668463ca3a29c904216703e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/d253e782065a4f0a964fde24523fdef5, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a1cd188798364399969a78d169f3313d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/eac04a1a34ab44d297fe802833bf5ea7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/0b0b89715a1d4fb58f4824d7e1c36b18, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/1545a1fcced04f5f8e3db7f8d4762774, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/6cbf881a5d794d349cc020b469e442e8, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/3f360c8d1741407c931cd96f2aa3e44a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/e5f43d48d3604375ab436f7854998894] to archive 2024-11-15T22:17:53,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:17:53,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/5f159bf16a7741a480568d465cee4735 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/5f159bf16a7741a480568d465cee4735 2024-11-15T22:17:53,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/81b0c49a07fd434e88cd09caae26384e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/81b0c49a07fd434e88cd09caae26384e 2024-11-15T22:17:53,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/7b9304b1859844fcad93ece6375d9bb2 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/7b9304b1859844fcad93ece6375d9bb2 2024-11-15T22:17:53,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/9ab86da33aad4d77816864f68b0e4500 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/9ab86da33aad4d77816864f68b0e4500 2024-11-15T22:17:53,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/3c480bedb22d432fa387de393c9a6708 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/3c480bedb22d432fa387de393c9a6708 2024-11-15T22:17:53,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ffb964f0814846a188e23c5abce985d7 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ffb964f0814846a188e23c5abce985d7 2024-11-15T22:17:53,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/c18f8810557a48cb819f4d62c7ddade6 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/c18f8810557a48cb819f4d62c7ddade6 2024-11-15T22:17:53,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ce7fde65df4446e3a3482a7c47d1b225 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ce7fde65df4446e3a3482a7c47d1b225 2024-11-15T22:17:53,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/dd1d83b6ab5042cfafd3a0d01b33d874 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/dd1d83b6ab5042cfafd3a0d01b33d874 2024-11-15T22:17:53,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a4fd6617e989408ba3c64b4916f488bd to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a4fd6617e989408ba3c64b4916f488bd 2024-11-15T22:17:53,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/947f158d5d95482c9a8b07f42455bb75 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/947f158d5d95482c9a8b07f42455bb75 2024-11-15T22:17:53,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/d26a30c9c6cf479db9a4840680867489 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/d26a30c9c6cf479db9a4840680867489 2024-11-15T22:17:53,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ed0af1a583bc4edba080c071257cdd51 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/ed0af1a583bc4edba080c071257cdd51 2024-11-15T22:17:53,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/c1f587620668463ca3a29c904216703e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/c1f587620668463ca3a29c904216703e 2024-11-15T22:17:53,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/d253e782065a4f0a964fde24523fdef5 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/d253e782065a4f0a964fde24523fdef5 2024-11-15T22:17:53,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a1cd188798364399969a78d169f3313d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/a1cd188798364399969a78d169f3313d 2024-11-15T22:17:53,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/eac04a1a34ab44d297fe802833bf5ea7 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/eac04a1a34ab44d297fe802833bf5ea7 2024-11-15T22:17:53,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/0b0b89715a1d4fb58f4824d7e1c36b18 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/0b0b89715a1d4fb58f4824d7e1c36b18 2024-11-15T22:17:53,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/1545a1fcced04f5f8e3db7f8d4762774 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/1545a1fcced04f5f8e3db7f8d4762774 2024-11-15T22:17:53,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/6cbf881a5d794d349cc020b469e442e8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/6cbf881a5d794d349cc020b469e442e8 2024-11-15T22:17:53,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/3f360c8d1741407c931cd96f2aa3e44a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/3f360c8d1741407c931cd96f2aa3e44a 2024-11-15T22:17:53,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/e5f43d48d3604375ab436f7854998894 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/e5f43d48d3604375ab436f7854998894 2024-11-15T22:17:53,430 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/recovered.edits/527.seqid, newMaxSeqId=527, maxSeqId=1 2024-11-15T22:17:53,430 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. 2024-11-15T22:17:53,431 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1676): Region close journal for 00af3e32e9b483f8e15f8f4708045315: Waiting for close lock at 1731709072481Running coprocessor pre-close hooks at 1731709072481Disabling compacts and flushes for region at 1731709072481Disabling writes for close at 1731709072481Obtaining lock to block concurrent updates at 1731709072481Preparing flush snapshotting stores in 00af3e32e9b483f8e15f8f4708045315 at 1731709072481Finished memstore snapshotting TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731709072482 (+1 ms)Flushing stores of TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315. at 1731709072482Flushing 00af3e32e9b483f8e15f8f4708045315/A: creating writer at 1731709072482Flushing 00af3e32e9b483f8e15f8f4708045315/A: appending metadata at 1731709072484 (+2 ms)Flushing 00af3e32e9b483f8e15f8f4708045315/A: closing flushed file at 1731709072484Flushing 00af3e32e9b483f8e15f8f4708045315/B: creating writer at 1731709072898 (+414 ms)Flushing 00af3e32e9b483f8e15f8f4708045315/B: appending metadata at 1731709072903 (+5 ms)Flushing 00af3e32e9b483f8e15f8f4708045315/B: closing flushed file at 1731709072903Flushing 00af3e32e9b483f8e15f8f4708045315/C: creating writer at 1731709073313 (+410 ms)Flushing 00af3e32e9b483f8e15f8f4708045315/C: appending metadata at 1731709073315 (+2 ms)Flushing 00af3e32e9b483f8e15f8f4708045315/C: closing flushed file at 1731709073315Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f02ad3e: reopening flushed file at 1731709073329 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6668d920: reopening flushed file at 1731709073333 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b3d6e2a: reopening flushed file at 1731709073338 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 00af3e32e9b483f8e15f8f4708045315 in 862ms, sequenceid=524, compaction requested=true at 1731709073343 (+5 ms)Writing region close event to WAL at 1731709073426 (+83 ms)Running coprocessor post-close hooks at 1731709073430 (+4 ms)Closed at 1731709073430 2024-11-15T22:17:53,432 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(157): Closed 00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:53,433 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=00af3e32e9b483f8e15f8f4708045315, regionState=CLOSED 2024-11-15T22:17:53,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:17:53,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-15T22:17:53,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseRegionProcedure 00af3e32e9b483f8e15f8f4708045315, server=e611192d6313,38779,1731708927034 in 1.1050 sec 2024-11-15T22:17:53,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-15T22:17:53,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00af3e32e9b483f8e15f8f4708045315, UNASSIGN in 1.1130 sec 2024-11-15T22:17:53,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-15T22:17:53,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1150 sec 2024-11-15T22:17:53,439 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731709073439"}]},"ts":"1731709073439"} 2024-11-15T22:17:53,441 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T22:17:53,441 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T22:17:53,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1250 sec 2024-11-15T22:17:53,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T22:17:53,451 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:53,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T22:17:53,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:53,453 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:53,454 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=131, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:53,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T22:17:53,456 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:53,457 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/recovered.edits] 2024-11-15T22:17:53,460 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/641cfe0fc55a4c9e906056e5069484e9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/641cfe0fc55a4c9e906056e5069484e9 2024-11-15T22:17:53,461 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/aba7df4ac9da433f940c47f92c80bc02 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/aba7df4ac9da433f940c47f92c80bc02 2024-11-15T22:17:53,462 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/b98281259a404c0ca16df8fb20aae01c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/b98281259a404c0ca16df8fb20aae01c 2024-11-15T22:17:53,462 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/caead73866ce4313bd91572a3a1ad879 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/A/caead73866ce4313bd91572a3a1ad879 2024-11-15T22:17:53,464 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/520386171c21425eb5eef6232bcc1711 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/520386171c21425eb5eef6232bcc1711 2024-11-15T22:17:53,465 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/90b64ce1f8374a46b03044e285182918 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/90b64ce1f8374a46b03044e285182918 2024-11-15T22:17:53,465 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/f2c1c9d26d88467e8593ba87150dcd2e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/f2c1c9d26d88467e8593ba87150dcd2e 2024-11-15T22:17:53,466 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/fccebae66d8344eb9e5f29d5b4161f8e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/B/fccebae66d8344eb9e5f29d5b4161f8e 2024-11-15T22:17:53,469 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/149ad284c1ce420ea195849de789f50e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/149ad284c1ce420ea195849de789f50e 2024-11-15T22:17:53,470 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/2509d8c49c0346d3a6ccf76784849f02 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/2509d8c49c0346d3a6ccf76784849f02 2024-11-15T22:17:53,471 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/41a8e57cdf404e6294708c2c64a5a236 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/41a8e57cdf404e6294708c2c64a5a236 2024-11-15T22:17:53,472 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/7078a879409840a39fea79f1f13cbe95 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/C/7078a879409840a39fea79f1f13cbe95 2024-11-15T22:17:53,474 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/recovered.edits/527.seqid to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315/recovered.edits/527.seqid 2024-11-15T22:17:53,475 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/00af3e32e9b483f8e15f8f4708045315 2024-11-15T22:17:53,475 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T22:17:53,477 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=131, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:53,481 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T22:17:53,485 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T22:17:53,486 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=131, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:53,486 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T22:17:53,487 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731709073486"}]},"ts":"9223372036854775807"} 2024-11-15T22:17:53,489 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T22:17:53,489 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 00af3e32e9b483f8e15f8f4708045315, NAME => 'TestAcidGuarantees,,1731709040793.00af3e32e9b483f8e15f8f4708045315.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T22:17:53,489 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T22:17:53,489 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731709073489"}]},"ts":"9223372036854775807"} 2024-11-15T22:17:53,493 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T22:17:53,494 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=131, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:53,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-15T22:17:53,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T22:17:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T22:17:53,564 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:53,571 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248 (was 251), OpenFileDescriptor=441 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1291 (was 1187) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=649 (was 251) - AvailableMemoryMB LEAK? - 2024-11-15T22:17:53,589 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=1291, ProcessCount=11, AvailableMemoryMB=648 2024-11-15T22:17:53,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:17:53,591 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:17:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:53,594 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T22:17:53,594 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:53,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 132 2024-11-15T22:17:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-15T22:17:53,595 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T22:17:53,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742367_1543 (size=1061) 2024-11-15T22:17:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-15T22:17:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-15T22:17:54,018 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e 2024-11-15T22:17:54,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742368_1544 (size=53) 2024-11-15T22:17:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-15T22:17:54,445 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:17:54,445 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e1d3c3e92f92bf8e015eabb60b759778, disabling compactions & flushes 2024-11-15T22:17:54,445 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:54,445 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:54,445 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. after waiting 0 ms 2024-11-15T22:17:54,445 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:54,445 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:54,445 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e1d3c3e92f92bf8e015eabb60b759778: Waiting for close lock at 1731709074445Disabling compacts and flushes for region at 1731709074445Disabling writes for close at 1731709074445Writing region close event to WAL at 1731709074445Closed at 1731709074445 2024-11-15T22:17:54,447 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T22:17:54,447 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731709074447"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731709074447"}]},"ts":"1731709074447"} 2024-11-15T22:17:54,448 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T22:17:54,449 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T22:17:54,449 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731709074449"}]},"ts":"1731709074449"} 2024-11-15T22:17:54,450 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T22:17:54,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1d3c3e92f92bf8e015eabb60b759778, ASSIGN}] 2024-11-15T22:17:54,452 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1d3c3e92f92bf8e015eabb60b759778, ASSIGN 2024-11-15T22:17:54,454 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1d3c3e92f92bf8e015eabb60b759778, ASSIGN; state=OFFLINE, location=e611192d6313,38779,1731708927034; forceNewPlan=false, retain=false 2024-11-15T22:17:54,604 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=e1d3c3e92f92bf8e015eabb60b759778, regionState=OPENING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:17:54,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1d3c3e92f92bf8e015eabb60b759778, ASSIGN because future has completed 2024-11-15T22:17:54,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:17:54,651 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T22:17:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-15T22:17:54,760 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:54,760 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7752): Opening region: {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:17:54,760 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:54,760 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:17:54,761 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7794): checking encryption for e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:54,761 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7797): checking classloading for e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:54,763 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:54,767 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:17:54,767 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1d3c3e92f92bf8e015eabb60b759778 columnFamilyName A 2024-11-15T22:17:54,767 DEBUG [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:54,767 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.HStore(327): Store=e1d3c3e92f92bf8e015eabb60b759778/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:17:54,768 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:54,769 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:17:54,769 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1d3c3e92f92bf8e015eabb60b759778 columnFamilyName B 2024-11-15T22:17:54,769 DEBUG [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:54,769 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.HStore(327): Store=e1d3c3e92f92bf8e015eabb60b759778/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:17:54,769 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:54,770 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:17:54,770 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1d3c3e92f92bf8e015eabb60b759778 columnFamilyName C 2024-11-15T22:17:54,770 DEBUG [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:54,771 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.HStore(327): Store=e1d3c3e92f92bf8e015eabb60b759778/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:17:54,771 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:54,771 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1038): replaying wal for e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:54,772 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:54,772 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:54,775 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1048): stopping wal replay for e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:54,775 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1060): Cleaning up temporary data for e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:54,775 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:17:54,777 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1093): writing seq id for e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:54,778 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:17:54,779 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1114): Opened e1d3c3e92f92bf8e015eabb60b759778; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71446393, jitterRate=0.06463421881198883}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:17:54,779 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:54,779 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1006): Region open journal for e1d3c3e92f92bf8e015eabb60b759778: Running coprocessor pre-open hook at 1731709074761Writing region info on filesystem at 1731709074761Initializing all the Stores at 1731709074762 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731709074762Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731709074762Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731709074762Cleaning up temporary data from old regions at 1731709074775 (+13 ms)Running coprocessor post-open hooks at 1731709074779 (+4 ms)Region opened successfully at 1731709074779 2024-11-15T22:17:54,780 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., pid=134, masterSystemTime=1731709074757 2024-11-15T22:17:54,782 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:54,782 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:54,782 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=e1d3c3e92f92bf8e015eabb60b759778, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:17:54,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:17:54,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-15T22:17:54,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; OpenRegionProcedure e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 in 178 msec 2024-11-15T22:17:54,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-15T22:17:54,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1d3c3e92f92bf8e015eabb60b759778, ASSIGN in 335 msec 2024-11-15T22:17:54,789 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T22:17:54,789 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731709074789"}]},"ts":"1731709074789"} 2024-11-15T22:17:54,790 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T22:17:54,791 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T22:17:54,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2000 sec 2024-11-15T22:17:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-15T22:17:55,742 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:55,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695f7e1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:55,743 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:55,743 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:55,745 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7ef3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:55,747 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ce57d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:55,748 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:55,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:55,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:55,752 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T22:17:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is e611192d6313,34969,1731708925722 2024-11-15T22:17:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@13452765 2024-11-15T22:17:55,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T22:17:55,753 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T22:17:55,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:17:55,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:17:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T22:17:55,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742369_1545 (size=1097) 2024-11-15T22:17:56,207 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-15T22:17:56,207 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-15T22:17:56,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:17:56,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1d3c3e92f92bf8e015eabb60b759778, REOPEN/MOVE}] 2024-11-15T22:17:56,212 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1d3c3e92f92bf8e015eabb60b759778, REOPEN/MOVE 2024-11-15T22:17:56,212 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=e1d3c3e92f92bf8e015eabb60b759778, regionState=CLOSING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:17:56,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1d3c3e92f92bf8e015eabb60b759778, REOPEN/MOVE because future has completed 2024-11-15T22:17:56,214 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:17:56,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:17:56,366 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,366 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:17:56,366 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing e1d3c3e92f92bf8e015eabb60b759778, disabling compactions & flushes 2024-11-15T22:17:56,366 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:56,366 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:56,366 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. after waiting 0 ms 2024-11-15T22:17:56,366 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:56,369 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:17:56,369 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:56,369 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for e1d3c3e92f92bf8e015eabb60b759778: Waiting for close lock at 1731709076366Running coprocessor pre-close hooks at 1731709076366Disabling compacts and flushes for region at 1731709076366Disabling writes for close at 1731709076366Writing region close event to WAL at 1731709076366Running coprocessor post-close hooks at 1731709076369 (+3 ms)Closed at 1731709076369 2024-11-15T22:17:56,370 WARN [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionServer(3299): Not adding moved region record: e1d3c3e92f92bf8e015eabb60b759778 to self. 2024-11-15T22:17:56,371 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,372 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=e1d3c3e92f92bf8e015eabb60b759778, regionState=CLOSED 2024-11-15T22:17:56,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:17:56,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-15T22:17:56,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 in 159 msec 2024-11-15T22:17:56,375 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1d3c3e92f92bf8e015eabb60b759778, REOPEN/MOVE; state=CLOSED, location=e611192d6313,38779,1731708927034; forceNewPlan=false, retain=true 2024-11-15T22:17:56,484 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T22:17:56,526 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=e1d3c3e92f92bf8e015eabb60b759778, regionState=OPENING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:17:56,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1d3c3e92f92bf8e015eabb60b759778, REOPEN/MOVE because future has completed 2024-11-15T22:17:56,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:17:56,682 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:56,682 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7752): Opening region: {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:17:56,683 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,683 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:17:56,683 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7794): checking encryption for e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,683 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7797): checking classloading for e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,684 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,684 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:17:56,685 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1d3c3e92f92bf8e015eabb60b759778 columnFamilyName A 2024-11-15T22:17:56,686 DEBUG [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:56,686 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.HStore(327): Store=e1d3c3e92f92bf8e015eabb60b759778/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:17:56,686 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,687 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:17:56,687 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1d3c3e92f92bf8e015eabb60b759778 columnFamilyName B 2024-11-15T22:17:56,687 DEBUG [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:56,688 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.HStore(327): Store=e1d3c3e92f92bf8e015eabb60b759778/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:17:56,688 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,688 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T22:17:56,688 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1d3c3e92f92bf8e015eabb60b759778 columnFamilyName C 2024-11-15T22:17:56,689 DEBUG [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:56,689 INFO [StoreOpener-e1d3c3e92f92bf8e015eabb60b759778-1 {}] regionserver.HStore(327): Store=e1d3c3e92f92bf8e015eabb60b759778/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:17:56,689 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:56,689 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1038): replaying wal for e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,690 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,690 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,691 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1048): stopping wal replay for e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,691 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1060): Cleaning up temporary data for e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,691 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:17:56,692 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1093): writing seq id for e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,693 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1114): Opened e1d3c3e92f92bf8e015eabb60b759778; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69049100, jitterRate=0.0289117693901062}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:17:56,693 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,694 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1006): Region open journal for e1d3c3e92f92bf8e015eabb60b759778: Running coprocessor pre-open hook at 1731709076683Writing region info on filesystem at 1731709076683Initializing all the Stores at 1731709076683Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731709076683Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731709076684 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731709076684Cleaning up temporary data from old regions at 1731709076691 (+7 ms)Running coprocessor post-open hooks at 1731709076693 (+2 ms)Region opened successfully at 1731709076694 (+1 ms) 2024-11-15T22:17:56,695 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., pid=139, masterSystemTime=1731709076680 2024-11-15T22:17:56,696 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:56,697 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:56,697 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=e1d3c3e92f92bf8e015eabb60b759778, regionState=OPEN, openSeqNum=5, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:17:56,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:17:56,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=137 2024-11-15T22:17:56,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 in 171 msec 2024-11-15T22:17:56,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-15T22:17:56,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1d3c3e92f92bf8e015eabb60b759778, REOPEN/MOVE in 490 msec 2024-11-15T22:17:56,704 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-15T22:17:56,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-11-15T22:17:56,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 947 msec 2024-11-15T22:17:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T22:17:56,707 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:17:56,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6919536d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:56,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:56,712 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:56,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fc2e5fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:56,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:56,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:56,714 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c214a71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d1887e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:56,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:56,717 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e84f602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:56,720 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:56,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@798c81fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2a1e31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:56,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:56,726 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@592b189a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:56,732 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:56,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55bada88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdeed38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,736 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:56,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:56,737 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:56,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:56,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:56,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72082aa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:56,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:56,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:56,739 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:56,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b73947, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b6eabbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:56,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:56,742 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:56,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:56,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:56,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50391121, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:56,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:56,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:56,744 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:56,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fbce8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b5dc35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:56,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:56,750 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38a57e9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:56,752 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffc2f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18d7368f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,753 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:56,753 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:56,754 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366e6f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:56,756 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ad93b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108b0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:56,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:56,765 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:56,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:56,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@605c4f18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:56,768 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:56,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72cb7628, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a5be7c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:56,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:56,770 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138d3966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:56,772 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b944bbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@162f1ee9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,34969,-1 for getting cluster id 2024-11-15T22:17:56,773 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:17:56,774 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6325f831-4157-4d7d-bab4-cef03871eef5' 2024-11-15T22:17:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:17:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6325f831-4157-4d7d-bab4-cef03871eef5" 2024-11-15T22:17:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66afcd4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,34969,-1] 2024-11-15T22:17:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:17:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:17:56,775 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:17:56,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e3697e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:17:56,780 DEBUG [Thread-2428 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:56,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:17:56,785 DEBUG [Thread-2432 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:56,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:56,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:56,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:56,788 DEBUG [Thread-2437 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:17:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:56,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:17:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-15T22:17:56,791 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:17:56,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-15T22:17:56,792 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:17:56,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:17:56,793 DEBUG [Thread-2430 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:17:56,794 DEBUG [Thread-2430 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:56,794 DEBUG [Thread-2430 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:56,794 DEBUG [Thread-2426 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:56,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:56,797 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:56,797 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:17:56,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:17:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1d3c3e92f92bf8e015eabb60b759778 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T22:17:56,800 DEBUG [Thread-2439 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:56,801 DEBUG [Thread-2445 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:56,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=A 2024-11-15T22:17:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:56,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:56,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=B 2024-11-15T22:17:56,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:56,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=C 2024-11-15T22:17:56,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:56,802 DEBUG [Thread-2434 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:56,802 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:56,803 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:17:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:56,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:17:56,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:56,807 DEBUG [Thread-2441 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:17:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:56,810 DEBUG [Thread-2443 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:17:56,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:56,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38779,1731708927034, seqNum=-1] 2024-11-15T22:17:56,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:17:56,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:17:56,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:17:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5] 2024-11-15T22:17:56,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709136826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709136827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:56,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:56,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709136828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:56,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:56,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709136835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:56,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709136835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:56,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a39339f121a94adc99519b3eef6edf9c_e1d3c3e92f92bf8e015eabb60b759778 is 50, key is test_row_0/A:col10/1731709076800/Put/seqid=0 2024-11-15T22:17:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-15T22:17:56,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742370_1546 (size=12154) 2024-11-15T22:17:56,913 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:56,920 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a39339f121a94adc99519b3eef6edf9c_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a39339f121a94adc99519b3eef6edf9c_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:56,924 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/2f8d01aefb1141bdbd349f94ec3a0105, store: [table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:17:56,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/2f8d01aefb1141bdbd349f94ec3a0105 is 175, key is test_row_0/A:col10/1731709076800/Put/seqid=0 2024-11-15T22:17:56,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709136944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:56,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-15T22:17:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:56,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:17:56,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:56,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:56,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:56,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709136944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:56,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709136945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:56,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:56,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709136953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:56,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742371_1547 (size=30955) 2024-11-15T22:17:56,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709136956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:56,968 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/2f8d01aefb1141bdbd349f94ec3a0105 2024-11-15T22:17:57,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/0b81c3981ab44da99dafd3b2387cc154 is 50, key is test_row_0/B:col10/1731709076800/Put/seqid=0 2024-11-15T22:17:57,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742372_1548 (size=12001) 2024-11-15T22:17:57,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/0b81c3981ab44da99dafd3b2387cc154 2024-11-15T22:17:57,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-15T22:17:57,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:57,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:17:57,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:57,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:57,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:57,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/67776852e377407d9fe2d6ac36afb53d is 50, key is test_row_0/C:col10/1731709076800/Put/seqid=0 2024-11-15T22:17:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-15T22:17:57,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742373_1549 (size=12001) 2024-11-15T22:17:57,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:57,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709137152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709137152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:57,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709137152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:57,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:57,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:57,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709137171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:57,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709137171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:57,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-15T22:17:57,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:57,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:17:57,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:57,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:57,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:57,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-15T22:17:57,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:57,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:17:57,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:57,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:57,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-15T22:17:57,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709137462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:57,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709137462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:57,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709137462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:57,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709137485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:57,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709137485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:57,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:57,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/67776852e377407d9fe2d6ac36afb53d 2024-11-15T22:17:57,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-15T22:17:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:17:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:57,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:17:57,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/2f8d01aefb1141bdbd349f94ec3a0105 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/2f8d01aefb1141bdbd349f94ec3a0105 2024-11-15T22:17:57,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/2f8d01aefb1141bdbd349f94ec3a0105, entries=150, sequenceid=16, filesize=30.2 K 2024-11-15T22:17:57,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/0b81c3981ab44da99dafd3b2387cc154 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/0b81c3981ab44da99dafd3b2387cc154 2024-11-15T22:17:57,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/0b81c3981ab44da99dafd3b2387cc154, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T22:17:57,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/67776852e377407d9fe2d6ac36afb53d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/67776852e377407d9fe2d6ac36afb53d 2024-11-15T22:17:57,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/67776852e377407d9fe2d6ac36afb53d, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T22:17:57,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e1d3c3e92f92bf8e015eabb60b759778 in 850ms, sequenceid=16, compaction requested=false 2024-11-15T22:17:57,650 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T22:17:57,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:17:57,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-15T22:17:57,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:57,731 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing e1d3c3e92f92bf8e015eabb60b759778 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T22:17:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=A 2024-11-15T22:17:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=B 2024-11-15T22:17:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=C 2024-11-15T22:17:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:57,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115765a07a6ccae48bab23815298bd4b2d3_e1d3c3e92f92bf8e015eabb60b759778 is 50, key is test_row_0/A:col10/1731709076827/Put/seqid=0 2024-11-15T22:17:57,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742374_1550 (size=12154) 2024-11-15T22:17:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-15T22:17:57,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:57,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:17:57,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:57,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709137986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:57,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:57,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709137986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:57,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:57,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709137987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:57,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:57,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:58,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:58,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709138001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:58,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709138001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:58,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:58,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709138101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:58,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709138102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:58,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709138102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:58,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:58,206 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115765a07a6ccae48bab23815298bd4b2d3_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115765a07a6ccae48bab23815298bd4b2d3_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:58,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/557982411faa429fb542b768ac625156, store: [table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:17:58,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/557982411faa429fb542b768ac625156 is 175, key is test_row_0/A:col10/1731709076827/Put/seqid=0 2024-11-15T22:17:58,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742375_1551 (size=30955) 2024-11-15T22:17:58,214 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/557982411faa429fb542b768ac625156 2024-11-15T22:17:58,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/1f0cb8b1e4e74d9ab9447da4ab0ad66d is 50, key is test_row_0/B:col10/1731709076827/Put/seqid=0 2024-11-15T22:17:58,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742376_1552 (size=12001) 2024-11-15T22:17:58,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709138311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:58,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:58,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709138311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709138311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:58,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709138621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709138621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709138622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:58,647 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/1f0cb8b1e4e74d9ab9447da4ab0ad66d 2024-11-15T22:17:58,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/bb4998b3187d42abb53f795bb8c5148a is 50, key is test_row_0/C:col10/1731709076827/Put/seqid=0 2024-11-15T22:17:58,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742377_1553 (size=12001) 2024-11-15T22:17:58,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-15T22:17:58,957 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T22:17:59,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:59,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709139011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:59,014 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in e1d3c3e92f92bf8e015eabb60b759778 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:17:59,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:59,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:59,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709139022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:59,024 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in e1d3c3e92f92bf8e015eabb60b759778 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:17:59,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:59,071 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/bb4998b3187d42abb53f795bb8c5148a 2024-11-15T22:17:59,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/557982411faa429fb542b768ac625156 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/557982411faa429fb542b768ac625156 2024-11-15T22:17:59,085 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/557982411faa429fb542b768ac625156, entries=150, sequenceid=36, filesize=30.2 K 2024-11-15T22:17:59,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/1f0cb8b1e4e74d9ab9447da4ab0ad66d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/1f0cb8b1e4e74d9ab9447da4ab0ad66d 2024-11-15T22:17:59,098 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/1f0cb8b1e4e74d9ab9447da4ab0ad66d, entries=150, sequenceid=36, filesize=11.7 K 2024-11-15T22:17:59,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/bb4998b3187d42abb53f795bb8c5148a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/bb4998b3187d42abb53f795bb8c5148a 2024-11-15T22:17:59,106 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/bb4998b3187d42abb53f795bb8c5148a, entries=150, sequenceid=36, filesize=11.7 K 2024-11-15T22:17:59,107 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e1d3c3e92f92bf8e015eabb60b759778 in 1375ms, sequenceid=36, compaction requested=false 2024-11-15T22:17:59,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:17:59,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:17:59,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-15T22:17:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-15T22:17:59,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-15T22:17:59,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3180 sec 2024-11-15T22:17:59,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 2.3230 sec 2024-11-15T22:17:59,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:59,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1d3c3e92f92bf8e015eabb60b759778 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T22:17:59,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=A 2024-11-15T22:17:59,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:59,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=B 2024-11-15T22:17:59,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:59,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=C 2024-11-15T22:17:59,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:17:59,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111541a28a6a1f014aa19868cb2ed3a612bd_e1d3c3e92f92bf8e015eabb60b759778 is 50, key is test_row_0/A:col10/1731709079134/Put/seqid=0 2024-11-15T22:17:59,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742378_1554 (size=12154) 2024-11-15T22:17:59,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709139163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:59,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709139163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:59,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709139167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:59,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:59,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709139271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:59,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709139271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:59,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:59,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709139282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:59,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:59,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709139481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:59,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709139481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:59,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709139491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:59,545 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:17:59,548 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111541a28a6a1f014aa19868cb2ed3a612bd_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111541a28a6a1f014aa19868cb2ed3a612bd_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:17:59,549 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/568ea4a27a894e1888b63ac93e9f4c6e, store: [table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:17:59,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/568ea4a27a894e1888b63ac93e9f4c6e is 175, key is test_row_0/A:col10/1731709079134/Put/seqid=0 2024-11-15T22:17:59,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742379_1555 (size=30955) 2024-11-15T22:17:59,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709139792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:59,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709139792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:59,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:17:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709139801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:17:59,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:17:59,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:17:59,953 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/568ea4a27a894e1888b63ac93e9f4c6e 2024-11-15T22:17:59,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/532786a2c612437abdd3e5ed81e3176b is 50, key is test_row_0/B:col10/1731709079134/Put/seqid=0 2024-11-15T22:17:59,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742380_1556 (size=12001) 2024-11-15T22:18:00,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709140304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:00,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709140304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:00,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709140311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:00,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/532786a2c612437abdd3e5ed81e3176b 2024-11-15T22:18:00,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/a9b327162090461faf101e5079ac374b is 50, key is test_row_0/C:col10/1731709079134/Put/seqid=0 2024-11-15T22:18:00,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742381_1557 (size=12001) 2024-11-15T22:18:00,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/a9b327162090461faf101e5079ac374b 2024-11-15T22:18:00,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/568ea4a27a894e1888b63ac93e9f4c6e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/568ea4a27a894e1888b63ac93e9f4c6e 2024-11-15T22:18:00,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/568ea4a27a894e1888b63ac93e9f4c6e, entries=150, sequenceid=50, filesize=30.2 K 2024-11-15T22:18:00,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/532786a2c612437abdd3e5ed81e3176b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/532786a2c612437abdd3e5ed81e3176b 2024-11-15T22:18:00,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/532786a2c612437abdd3e5ed81e3176b, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T22:18:00,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/a9b327162090461faf101e5079ac374b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/a9b327162090461faf101e5079ac374b 2024-11-15T22:18:00,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/a9b327162090461faf101e5079ac374b, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T22:18:00,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e1d3c3e92f92bf8e015eabb60b759778 in 1302ms, sequenceid=50, compaction requested=true 2024-11-15T22:18:00,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:00,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1d3c3e92f92bf8e015eabb60b759778:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:18:00,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:18:00,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1d3c3e92f92bf8e015eabb60b759778:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:18:00,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:18:00,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1d3c3e92f92bf8e015eabb60b759778:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:18:00,437 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:18:00,437 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:18:00,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:18:00,438 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:18:00,438 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): e1d3c3e92f92bf8e015eabb60b759778/B is initiating minor compaction (all files) 2024-11-15T22:18:00,438 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:18:00,439 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1d3c3e92f92bf8e015eabb60b759778/B in TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:00,439 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): e1d3c3e92f92bf8e015eabb60b759778/A is initiating minor compaction (all files) 2024-11-15T22:18:00,439 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1d3c3e92f92bf8e015eabb60b759778/A in TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:00,439 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/0b81c3981ab44da99dafd3b2387cc154, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/1f0cb8b1e4e74d9ab9447da4ab0ad66d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/532786a2c612437abdd3e5ed81e3176b] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp, totalSize=35.2 K 2024-11-15T22:18:00,439 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/2f8d01aefb1141bdbd349f94ec3a0105, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/557982411faa429fb542b768ac625156, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/568ea4a27a894e1888b63ac93e9f4c6e] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp, totalSize=90.7 K 2024-11-15T22:18:00,439 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:00,439 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/2f8d01aefb1141bdbd349f94ec3a0105, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/557982411faa429fb542b768ac625156, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/568ea4a27a894e1888b63ac93e9f4c6e] 2024-11-15T22:18:00,439 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b81c3981ab44da99dafd3b2387cc154, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731709076798 2024-11-15T22:18:00,440 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f8d01aefb1141bdbd349f94ec3a0105, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731709076798 2024-11-15T22:18:00,440 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f0cb8b1e4e74d9ab9447da4ab0ad66d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731709076814 2024-11-15T22:18:00,440 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 557982411faa429fb542b768ac625156, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731709076814 2024-11-15T22:18:00,440 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 568ea4a27a894e1888b63ac93e9f4c6e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731709077978 2024-11-15T22:18:00,440 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 532786a2c612437abdd3e5ed81e3176b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731709077978 2024-11-15T22:18:00,446 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:00,449 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1d3c3e92f92bf8e015eabb60b759778#B#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:18:00,449 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/12949ba19be446c78921e7adf2333109 is 50, key is test_row_0/B:col10/1731709079134/Put/seqid=0 2024-11-15T22:18:00,450 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411150b008a8e717749d5ab17ea4b0d86a5cf_e1d3c3e92f92bf8e015eabb60b759778 store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:00,452 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411150b008a8e717749d5ab17ea4b0d86a5cf_e1d3c3e92f92bf8e015eabb60b759778, store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:00,452 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150b008a8e717749d5ab17ea4b0d86a5cf_e1d3c3e92f92bf8e015eabb60b759778 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:00,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742382_1558 (size=12104) 2024-11-15T22:18:00,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742383_1559 (size=4469) 2024-11-15T22:18:00,470 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1d3c3e92f92bf8e015eabb60b759778#A#compaction#476 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:18:00,471 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/8d75eb1d9b6241d3a74539a297ee7c76 is 175, key is test_row_0/A:col10/1731709079134/Put/seqid=0 2024-11-15T22:18:00,473 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/12949ba19be446c78921e7adf2333109 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/12949ba19be446c78921e7adf2333109 2024-11-15T22:18:00,477 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1d3c3e92f92bf8e015eabb60b759778/B of e1d3c3e92f92bf8e015eabb60b759778 into 12949ba19be446c78921e7adf2333109(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:18:00,477 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:00,477 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., storeName=e1d3c3e92f92bf8e015eabb60b759778/B, priority=13, startTime=1731709080437; duration=0sec 2024-11-15T22:18:00,477 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:18:00,477 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1d3c3e92f92bf8e015eabb60b759778:B 2024-11-15T22:18:00,477 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:18:00,478 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:18:00,478 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): e1d3c3e92f92bf8e015eabb60b759778/C is initiating minor compaction (all files) 2024-11-15T22:18:00,478 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1d3c3e92f92bf8e015eabb60b759778/C in TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:00,478 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/67776852e377407d9fe2d6ac36afb53d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/bb4998b3187d42abb53f795bb8c5148a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/a9b327162090461faf101e5079ac374b] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp, totalSize=35.2 K 2024-11-15T22:18:00,479 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 67776852e377407d9fe2d6ac36afb53d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731709076798 2024-11-15T22:18:00,480 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting bb4998b3187d42abb53f795bb8c5148a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731709076814 2024-11-15T22:18:00,480 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting a9b327162090461faf101e5079ac374b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731709077978 2024-11-15T22:18:00,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742384_1560 (size=31058) 2024-11-15T22:18:00,503 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1d3c3e92f92bf8e015eabb60b759778#C#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:18:00,506 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/d5fd084958b2454398e9a17758e6b95a is 50, key is test_row_0/C:col10/1731709079134/Put/seqid=0 2024-11-15T22:18:00,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742385_1561 (size=12104) 2024-11-15T22:18:00,900 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/8d75eb1d9b6241d3a74539a297ee7c76 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/8d75eb1d9b6241d3a74539a297ee7c76 2024-11-15T22:18:00,904 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1d3c3e92f92bf8e015eabb60b759778/A of e1d3c3e92f92bf8e015eabb60b759778 into 8d75eb1d9b6241d3a74539a297ee7c76(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:18:00,904 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:00,904 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., storeName=e1d3c3e92f92bf8e015eabb60b759778/A, priority=13, startTime=1731709080437; duration=0sec 2024-11-15T22:18:00,904 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:18:00,904 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1d3c3e92f92bf8e015eabb60b759778:A 2024-11-15T22:18:00,917 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/d5fd084958b2454398e9a17758e6b95a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/d5fd084958b2454398e9a17758e6b95a 2024-11-15T22:18:00,923 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1d3c3e92f92bf8e015eabb60b759778/C of e1d3c3e92f92bf8e015eabb60b759778 into d5fd084958b2454398e9a17758e6b95a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:18:00,923 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:00,923 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., storeName=e1d3c3e92f92bf8e015eabb60b759778/C, priority=13, startTime=1731709080437; duration=0sec 2024-11-15T22:18:00,923 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:18:00,923 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1d3c3e92f92bf8e015eabb60b759778:C 2024-11-15T22:18:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-15T22:18:00,951 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:18:00,952 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:18:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-15T22:18:00,953 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:18:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T22:18:00,954 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:18:00,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:18:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:01,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1d3c3e92f92bf8e015eabb60b759778 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T22:18:01,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=A 2024-11-15T22:18:01,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:01,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=B 2024-11-15T22:18:01,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:01,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=C 2024-11-15T22:18:01,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:01,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158862d7d942204301952b3a4a25a7a198_e1d3c3e92f92bf8e015eabb60b759778 is 50, key is test_row_0/A:col10/1731709079159/Put/seqid=0 2024-11-15T22:18:01,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742386_1562 (size=14594) 2024-11-15T22:18:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T22:18:01,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709141058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:01,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709141058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:01,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-15T22:18:01,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:01,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:01,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:01,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709141172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:01,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709141172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:01,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-15T22:18:01,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:01,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:01,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:01,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T22:18:01,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709141311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:01,314 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in e1d3c3e92f92bf8e015eabb60b759778 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:18:01,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:01,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709141321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:01,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709141321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:01,322 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in e1d3c3e92f92bf8e015eabb60b759778 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:18:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:01,322 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in e1d3c3e92f92bf8e015eabb60b759778 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:18:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:01,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709141381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:01,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:01,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709141381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:01,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:01,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-15T22:18:01,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:01,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:01,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:01,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,446 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:01,450 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158862d7d942204301952b3a4a25a7a198_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158862d7d942204301952b3a4a25a7a198_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:01,450 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/1d28b5cec2d64411b6d6d0de816d1a06, store: [table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:01,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/1d28b5cec2d64411b6d6d0de816d1a06 is 175, key is test_row_0/A:col10/1731709079159/Put/seqid=0 2024-11-15T22:18:01,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742387_1563 (size=39549) 2024-11-15T22:18:01,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-15T22:18:01,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:01,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:01,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:01,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T22:18:01,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709141692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:01,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709141692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:01,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-15T22:18:01,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:01,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:01,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:01,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,857 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=72, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/1d28b5cec2d64411b6d6d0de816d1a06 2024-11-15T22:18:01,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/37365b47305c4f57accec674165bb04d is 50, key is test_row_0/B:col10/1731709079159/Put/seqid=0 2024-11-15T22:18:01,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742388_1564 (size=12001) 2024-11-15T22:18:01,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-15T22:18:01,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:01,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:01,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:01,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:01,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:02,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-15T22:18:02,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:02,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:02,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:02,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:02,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T22:18:02,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-15T22:18:02,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:02,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:02,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:02,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:02,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:02,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709142202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:02,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709142202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:02,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:02,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:02,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:02,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/37365b47305c4f57accec674165bb04d 2024-11-15T22:18:02,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/b1a32ffd93ad4782a5c5da8ea7d32a70 is 50, key is test_row_0/C:col10/1731709079159/Put/seqid=0 2024-11-15T22:18:02,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742389_1565 (size=12001) 2024-11-15T22:18:02,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/b1a32ffd93ad4782a5c5da8ea7d32a70 2024-11-15T22:18:02,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/1d28b5cec2d64411b6d6d0de816d1a06 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/1d28b5cec2d64411b6d6d0de816d1a06 2024-11-15T22:18:02,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/1d28b5cec2d64411b6d6d0de816d1a06, entries=200, sequenceid=72, filesize=38.6 K 2024-11-15T22:18:02,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/37365b47305c4f57accec674165bb04d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/37365b47305c4f57accec674165bb04d 2024-11-15T22:18:02,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/37365b47305c4f57accec674165bb04d, entries=150, sequenceid=72, filesize=11.7 K 2024-11-15T22:18:02,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/b1a32ffd93ad4782a5c5da8ea7d32a70 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/b1a32ffd93ad4782a5c5da8ea7d32a70 2024-11-15T22:18:02,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/b1a32ffd93ad4782a5c5da8ea7d32a70, entries=150, sequenceid=72, filesize=11.7 K 2024-11-15T22:18:02,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e1d3c3e92f92bf8e015eabb60b759778 in 1302ms, sequenceid=72, compaction requested=false 2024-11-15T22:18:02,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:02,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-15T22:18:02,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:02,341 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing e1d3c3e92f92bf8e015eabb60b759778 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T22:18:02,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=A 2024-11-15T22:18:02,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:02,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=B 2024-11-15T22:18:02,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:02,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=C 2024-11-15T22:18:02,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:02,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b64815c2bf9f48ff8f78117211a0b5cb_e1d3c3e92f92bf8e015eabb60b759778 is 50, key is test_row_0/A:col10/1731709081057/Put/seqid=0 2024-11-15T22:18:02,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742390_1566 (size=12154) 2024-11-15T22:18:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:02,374 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b64815c2bf9f48ff8f78117211a0b5cb_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b64815c2bf9f48ff8f78117211a0b5cb_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:02,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/9c54bc69e0d4465c948a8dcabf7745c7, store: [table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:02,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/9c54bc69e0d4465c948a8dcabf7745c7 is 175, key is test_row_0/A:col10/1731709081057/Put/seqid=0 2024-11-15T22:18:02,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742391_1567 (size=30955) 2024-11-15T22:18:02,781 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/9c54bc69e0d4465c948a8dcabf7745c7 2024-11-15T22:18:02,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/4ca7c30c865047b0880d5913267eafe9 is 50, key is test_row_0/B:col10/1731709081057/Put/seqid=0 2024-11-15T22:18:02,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742392_1568 (size=12001) 2024-11-15T22:18:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T22:18:03,196 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/4ca7c30c865047b0880d5913267eafe9 2024-11-15T22:18:03,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/ed64efa3335545ce8f58de7a9db12e93 is 50, key is test_row_0/C:col10/1731709081057/Put/seqid=0 2024-11-15T22:18:03,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742393_1569 (size=12001) 2024-11-15T22:18:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:03,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:03,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709143257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:03,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709143258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:03,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709143322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:03,323 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_1' in e1d3c3e92f92bf8e015eabb60b759778 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:18:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:03,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709143331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:03,336 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in e1d3c3e92f92bf8e015eabb60b759778 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:18:03,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:03,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709143347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:03,350 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_2' in e1d3c3e92f92bf8e015eabb60b759778 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:18:03,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:03,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709143372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:03,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709143372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:03,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709143581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:03,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709143581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:03,614 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/ed64efa3335545ce8f58de7a9db12e93 2024-11-15T22:18:03,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/9c54bc69e0d4465c948a8dcabf7745c7 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/9c54bc69e0d4465c948a8dcabf7745c7 2024-11-15T22:18:03,622 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/9c54bc69e0d4465c948a8dcabf7745c7, entries=150, sequenceid=85, filesize=30.2 K 2024-11-15T22:18:03,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/4ca7c30c865047b0880d5913267eafe9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/4ca7c30c865047b0880d5913267eafe9 2024-11-15T22:18:03,627 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/4ca7c30c865047b0880d5913267eafe9, entries=150, sequenceid=85, filesize=11.7 K 2024-11-15T22:18:03,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/ed64efa3335545ce8f58de7a9db12e93 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/ed64efa3335545ce8f58de7a9db12e93 2024-11-15T22:18:03,637 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/ed64efa3335545ce8f58de7a9db12e93, entries=150, sequenceid=85, filesize=11.7 K 2024-11-15T22:18:03,638 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e1d3c3e92f92bf8e015eabb60b759778 in 1297ms, sequenceid=85, compaction requested=true 2024-11-15T22:18:03,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:03,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:03,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-15T22:18:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-15T22:18:03,644 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-15T22:18:03,644 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6860 sec 2024-11-15T22:18:03,646 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.6930 sec 2024-11-15T22:18:03,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:03,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1d3c3e92f92bf8e015eabb60b759778 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T22:18:03,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=A 2024-11-15T22:18:03,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:03,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=B 2024-11-15T22:18:03,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:03,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=C 2024-11-15T22:18:03,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:03,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a25100afa37a4ebf9d3fae46d6fcfdf9_e1d3c3e92f92bf8e015eabb60b759778 is 50, key is test_row_0/A:col10/1731709083894/Put/seqid=0 2024-11-15T22:18:03,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:03,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709143925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:03,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709143927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:03,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742394_1570 (size=14594) 2024-11-15T22:18:04,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709144032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:04,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709144041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:04,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:04,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:04,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:04,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709144243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:04,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:04,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:04,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:04,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709144252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:04,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:04,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:04,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:04,340 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:04,343 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a25100afa37a4ebf9d3fae46d6fcfdf9_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a25100afa37a4ebf9d3fae46d6fcfdf9_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:04,344 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/6bab8206cf8440ffba76dcb144a80c2b, store: [table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:04,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/6bab8206cf8440ffba76dcb144a80c2b is 175, key is test_row_0/A:col10/1731709083894/Put/seqid=0 2024-11-15T22:18:04,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742395_1571 (size=39549) 2024-11-15T22:18:04,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709144552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:04,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:04,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:04,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:04,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709144561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:04,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:04,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:04,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:04,749 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/6bab8206cf8440ffba76dcb144a80c2b 2024-11-15T22:18:04,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/79f3ef3bf5194d9d88ce89dc303ac6ba is 50, key is test_row_0/B:col10/1731709083894/Put/seqid=0 2024-11-15T22:18:04,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742396_1572 (size=12001) 2024-11-15T22:18:05,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709145061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:05,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709145072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T22:18:05,112 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:18:05,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:18:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-15T22:18:05,114 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:18:05,114 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:18:05,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:18:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T22:18:05,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/79f3ef3bf5194d9d88ce89dc303ac6ba 2024-11-15T22:18:05,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/73775e49fc974727b57324e571220378 is 50, key is test_row_0/C:col10/1731709083894/Put/seqid=0 2024-11-15T22:18:05,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742397_1573 (size=12001) 2024-11-15T22:18:05,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/73775e49fc974727b57324e571220378 2024-11-15T22:18:05,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/6bab8206cf8440ffba76dcb144a80c2b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/6bab8206cf8440ffba76dcb144a80c2b 2024-11-15T22:18:05,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/6bab8206cf8440ffba76dcb144a80c2b, entries=200, sequenceid=105, filesize=38.6 K 2024-11-15T22:18:05,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/79f3ef3bf5194d9d88ce89dc303ac6ba as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/79f3ef3bf5194d9d88ce89dc303ac6ba 2024-11-15T22:18:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/79f3ef3bf5194d9d88ce89dc303ac6ba, entries=150, sequenceid=105, filesize=11.7 K 2024-11-15T22:18:05,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/73775e49fc974727b57324e571220378 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/73775e49fc974727b57324e571220378 2024-11-15T22:18:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/73775e49fc974727b57324e571220378, entries=150, sequenceid=105, filesize=11.7 K 2024-11-15T22:18:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e1d3c3e92f92bf8e015eabb60b759778 in 1321ms, sequenceid=105, compaction requested=true 2024-11-15T22:18:05,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:05,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1d3c3e92f92bf8e015eabb60b759778:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:18:05,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:18:05,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1d3c3e92f92bf8e015eabb60b759778:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:18:05,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:18:05,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1d3c3e92f92bf8e015eabb60b759778:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:18:05,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T22:18:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,217 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:18:05,217 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:18:05,218 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:18:05,218 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:18:05,218 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): e1d3c3e92f92bf8e015eabb60b759778/A is initiating minor compaction (all files) 2024-11-15T22:18:05,218 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): e1d3c3e92f92bf8e015eabb60b759778/C is initiating minor compaction (all files) 2024-11-15T22:18:05,218 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1d3c3e92f92bf8e015eabb60b759778/A in TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:05,218 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1d3c3e92f92bf8e015eabb60b759778/C in TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:05,218 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/8d75eb1d9b6241d3a74539a297ee7c76, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/1d28b5cec2d64411b6d6d0de816d1a06, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/9c54bc69e0d4465c948a8dcabf7745c7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/6bab8206cf8440ffba76dcb144a80c2b] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp, totalSize=137.8 K 2024-11-15T22:18:05,218 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/d5fd084958b2454398e9a17758e6b95a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/b1a32ffd93ad4782a5c5da8ea7d32a70, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/ed64efa3335545ce8f58de7a9db12e93, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/73775e49fc974727b57324e571220378] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp, totalSize=47.0 K 2024-11-15T22:18:05,218 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:05,218 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/8d75eb1d9b6241d3a74539a297ee7c76, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/1d28b5cec2d64411b6d6d0de816d1a06, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/9c54bc69e0d4465c948a8dcabf7745c7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/6bab8206cf8440ffba76dcb144a80c2b] 2024-11-15T22:18:05,219 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d75eb1d9b6241d3a74539a297ee7c76, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731709077978 2024-11-15T22:18:05,219 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting d5fd084958b2454398e9a17758e6b95a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731709077978 2024-11-15T22:18:05,219 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting b1a32ffd93ad4782a5c5da8ea7d32a70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1731709079159 2024-11-15T22:18:05,219 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d28b5cec2d64411b6d6d0de816d1a06, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1731709079159 2024-11-15T22:18:05,219 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting ed64efa3335545ce8f58de7a9db12e93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731709081052 2024-11-15T22:18:05,219 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c54bc69e0d4465c948a8dcabf7745c7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731709081052 2024-11-15T22:18:05,219 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 73775e49fc974727b57324e571220378, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731709083250 2024-11-15T22:18:05,220 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bab8206cf8440ffba76dcb144a80c2b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731709083250 2024-11-15T22:18:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T22:18:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,228 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1d3c3e92f92bf8e015eabb60b759778#C#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:18:05,228 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/188fcd7c59be4c6a8136f854c82d8899 is 50, key is test_row_0/C:col10/1731709083894/Put/seqid=0 2024-11-15T22:18:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,235 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,237 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115ae479162cefc475bb66d2914c1fc48a8_e1d3c3e92f92bf8e015eabb60b759778 store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,240 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115ae479162cefc475bb66d2914c1fc48a8_e1d3c3e92f92bf8e015eabb60b759778, store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,241 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ae479162cefc475bb66d2914c1fc48a8_e1d3c3e92f92bf8e015eabb60b759778 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742399_1575 (size=4469) 2024-11-15T22:18:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742398_1574 (size=12241) 2024-11-15T22:18:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,258 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/188fcd7c59be4c6a8136f854c82d8899 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/188fcd7c59be4c6a8136f854c82d8899 2024-11-15T22:18:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,265 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1d3c3e92f92bf8e015eabb60b759778/C of e1d3c3e92f92bf8e015eabb60b759778 into 188fcd7c59be4c6a8136f854c82d8899(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:18:05,265 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:05,265 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., storeName=e1d3c3e92f92bf8e015eabb60b759778/C, priority=12, startTime=1731709085217; duration=0sec 2024-11-15T22:18:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,265 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:18:05,265 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1d3c3e92f92bf8e015eabb60b759778:C 2024-11-15T22:18:05,265 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:18:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-15T22:18:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,267 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing e1d3c3e92f92bf8e015eabb60b759778 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T22:18:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=A 2024-11-15T22:18:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=B 2024-11-15T22:18:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=C 2024-11-15T22:18:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:05,268 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:18:05,268 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): e1d3c3e92f92bf8e015eabb60b759778/B is initiating minor compaction (all files) 2024-11-15T22:18:05,268 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1d3c3e92f92bf8e015eabb60b759778/B in TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:05,268 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/12949ba19be446c78921e7adf2333109, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/37365b47305c4f57accec674165bb04d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/4ca7c30c865047b0880d5913267eafe9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/79f3ef3bf5194d9d88ce89dc303ac6ba] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp, totalSize=47.0 K 2024-11-15T22:18:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,269 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 12949ba19be446c78921e7adf2333109, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731709077978 2024-11-15T22:18:05,270 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 37365b47305c4f57accec674165bb04d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1731709079159 2024-11-15T22:18:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,271 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ca7c30c865047b0880d5913267eafe9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731709081052 2024-11-15T22:18:05,271 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 79f3ef3bf5194d9d88ce89dc303ac6ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731709083250 2024-11-15T22:18:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111592f4ca0cecb749e4a4e8530dddecd0ef_e1d3c3e92f92bf8e015eabb60b759778 is 50, key is test_row_1/A:col10/1731709083924/Put/seqid=0 2024-11-15T22:18:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742400_1576 (size=9714) 2024-11-15T22:18:05,285 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1d3c3e92f92bf8e015eabb60b759778#B#compaction#491 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:18:05,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,286 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/11e205a18907412db2346a6260843b65 is 50, key is test_row_0/B:col10/1731709083894/Put/seqid=0 2024-11-15T22:18:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,289 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111592f4ca0cecb749e4a4e8530dddecd0ef_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111592f4ca0cecb749e4a4e8530dddecd0ef_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/3c7cdda8de594b85a0df1c19b2bc7282, store: [table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:05,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/3c7cdda8de594b85a0df1c19b2bc7282 is 175, key is test_row_1/A:col10/1731709083924/Put/seqid=0 2024-11-15T22:18:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742401_1577 (size=12241) 2024-11-15T22:18:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742402_1578 (size=22361) 2024-11-15T22:18:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T22:18:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,658 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1d3c3e92f92bf8e015eabb60b759778#A#compaction#489 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:18:05,658 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/00e265a869a2407fb346ee7a33abbed6 is 175, key is test_row_0/A:col10/1731709083894/Put/seqid=0 2024-11-15T22:18:05,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742403_1579 (size=31195) 2024-11-15T22:18:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,673 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/00e265a869a2407fb346ee7a33abbed6 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/00e265a869a2407fb346ee7a33abbed6 2024-11-15T22:18:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,678 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1d3c3e92f92bf8e015eabb60b759778/A of e1d3c3e92f92bf8e015eabb60b759778 into 00e265a869a2407fb346ee7a33abbed6(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:18:05,678 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:05,678 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., storeName=e1d3c3e92f92bf8e015eabb60b759778/A, priority=12, startTime=1731709085216; duration=0sec 2024-11-15T22:18:05,678 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:18:05,678 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1d3c3e92f92bf8e015eabb60b759778:A 2024-11-15T22:18:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,721 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/11e205a18907412db2346a6260843b65 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/11e205a18907412db2346a6260843b65 2024-11-15T22:18:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,728 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1d3c3e92f92bf8e015eabb60b759778/B of e1d3c3e92f92bf8e015eabb60b759778 into 11e205a18907412db2346a6260843b65(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:18:05,728 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:05,728 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., storeName=e1d3c3e92f92bf8e015eabb60b759778/B, priority=12, startTime=1731709085216; duration=0sec 2024-11-15T22:18:05,728 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:18:05,728 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1d3c3e92f92bf8e015eabb60b759778:B 2024-11-15T22:18:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T22:18:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,755 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/3c7cdda8de594b85a0df1c19b2bc7282 2024-11-15T22:18:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/ad2cc05aac9b48bc92ad1a1b3eb2adba is 50, key is test_row_1/B:col10/1731709083924/Put/seqid=0 2024-11-15T22:18:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742404_1580 (size=9657) 2024-11-15T22:18:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,826 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/ad2cc05aac9b48bc92ad1a1b3eb2adba 2024-11-15T22:18:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/adc49a21b5ee422eac145c41541c9f6e is 50, key is test_row_1/C:col10/1731709083924/Put/seqid=0 2024-11-15T22:18:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742405_1581 (size=9657) 2024-11-15T22:18:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:06,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709146146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:06,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709146151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:06,251 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/adc49a21b5ee422eac145c41541c9f6e 2024-11-15T22:18:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T22:18:06,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/3c7cdda8de594b85a0df1c19b2bc7282 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/3c7cdda8de594b85a0df1c19b2bc7282 2024-11-15T22:18:06,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709146261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:06,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709146263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:06,269 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/3c7cdda8de594b85a0df1c19b2bc7282, entries=100, sequenceid=118, filesize=21.8 K 2024-11-15T22:18:06,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/ad2cc05aac9b48bc92ad1a1b3eb2adba as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/ad2cc05aac9b48bc92ad1a1b3eb2adba 2024-11-15T22:18:06,297 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/ad2cc05aac9b48bc92ad1a1b3eb2adba, entries=100, sequenceid=118, filesize=9.4 K 2024-11-15T22:18:06,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/adc49a21b5ee422eac145c41541c9f6e as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/adc49a21b5ee422eac145c41541c9f6e 2024-11-15T22:18:06,304 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/adc49a21b5ee422eac145c41541c9f6e, entries=100, sequenceid=118, filesize=9.4 K 2024-11-15T22:18:06,309 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e1d3c3e92f92bf8e015eabb60b759778 in 1042ms, sequenceid=118, compaction requested=false 2024-11-15T22:18:06,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:06,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:06,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-15T22:18:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-15T22:18:06,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-15T22:18:06,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2010 sec 2024-11-15T22:18:06,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.2160 sec 2024-11-15T22:18:06,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731709086129, max timestamp=1731709086475Num uniques 859; 2024-11-15T22:18:06,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731709086129, max timestamp=1731709086475Num uniques 859; 2024-11-15T22:18:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:06,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:18:06,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:18:06,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731709086129, max timestamp=1731709086474Num uniques 859; 2024-11-15T22:18:06,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:18:06,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1d3c3e92f92bf8e015eabb60b759778 3/3 column families, dataSize=33.54 KB heapSize=86.16 KB 2024-11-15T22:18:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=A 2024-11-15T22:18:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:18:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=B 2024-11-15T22:18:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:18:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=C 2024-11-15T22:18:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:18:06,487 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T22:18:06,487 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T22:18:06,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152c4f3a5c1f684d5f87208b68f1163d3b_e1d3c3e92f92bf8e015eabb60b759778 is 50, key is test_row_0/A:col10/1731709086144/Put/seqid=0 2024-11-15T22:18:06,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742406_1582 (size=12304) 2024-11-15T22:18:06,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731709086538, max timestamp=1731709086565Num uniques 859; 2024-11-15T22:18:06,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T22:18:06,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731709086538, max timestamp=1731709086565Num uniques 859; 2024-11-15T22:18:06,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T22:18:06,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731709086538, max timestamp=1731709086565Num uniques 859; 2024-11-15T22:18:06,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T22:18:06,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731709086608, max timestamp=1731709086645Num uniques 1009; 2024-11-15T22:18:06,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-15T22:18:06,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731709086608, max timestamp=1731709086645Num uniques 1009; 2024-11-15T22:18:06,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-15T22:18:06,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731709086608, max timestamp=1731709086645Num uniques 1009; 2024-11-15T22:18:06,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-15T22:18:06,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731709086696, max timestamp=1731709086741Num uniques 1077; 2024-11-15T22:18:06,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731709086696, max timestamp=1731709086741Num uniques 1077; 2024-11-15T22:18:06,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=6870, suffix heap size=139976,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=150 2024-11-15T22:18:06,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=6870, suffix heap size=139976,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=150 2024-11-15T22:18:06,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731709086696, max timestamp=1731709086741Num uniques 1077; 2024-11-15T22:18:06,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-15T22:18:06,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731709086789, max timestamp=1731709086882Num uniques 1036; 2024-11-15T22:18:06,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-15T22:18:06,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731709086789, max timestamp=1731709086882Num uniques 1009; 2024-11-15T22:18:06,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10806, suffix heap size=132904,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=236 2024-11-15T22:18:06,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731709086789, max timestamp=1731709086882Num uniques 1009; 2024-11-15T22:18:06,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10806, suffix heap size=132904,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=236 2024-11-15T22:18:06,956 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:06,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731709086954, max timestamp=1731709086982Num uniques 1095; 2024-11-15T22:18:06,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-15T22:18:06,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731709086954, max timestamp=1731709086982Num uniques 1095; 2024-11-15T22:18:06,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731709086954, max timestamp=1731709086982Num uniques 1095; 2024-11-15T22:18:06,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-15T22:18:06,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-15T22:18:06,998 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152c4f3a5c1f684d5f87208b68f1163d3b_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152c4f3a5c1f684d5f87208b68f1163d3b_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:07,005 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/e32c0a4e7457406b9f580965253d25c9, store: [table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:07,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/e32c0a4e7457406b9f580965253d25c9 is 175, key is test_row_0/A:col10/1731709086144/Put/seqid=0 2024-11-15T22:18:07,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742407_1583 (size=31105) 2024-11-15T22:18:07,072 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=142, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/e32c0a4e7457406b9f580965253d25c9 2024-11-15T22:18:07,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731709087074, max timestamp=1731709087122Num uniques 1104; 2024-11-15T22:18:07,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731709087074, max timestamp=1731709087122Num uniques 1104; 2024-11-15T22:18:07,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-15T22:18:07,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-15T22:18:07,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731709087074, max timestamp=1731709087119Num uniques 1104; 2024-11-15T22:18:07,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-15T22:18:07,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/2b2a77fc2f704730903407520d015556 is 50, key is test_row_0/B:col10/1731709086144/Put/seqid=0 2024-11-15T22:18:07,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742408_1584 (size=12151) 2024-11-15T22:18:07,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731709087196, max timestamp=1731709087228Num uniques 1013; 2024-11-15T22:18:07,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-15T22:18:07,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731709087196, max timestamp=1731709087228Num uniques 1013; 2024-11-15T22:18:07,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-15T22:18:07,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731709087196, max timestamp=1731709087228Num uniques 1013; 2024-11-15T22:18:07,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-15T22:18:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T22:18:07,274 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:18:07,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:18:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-15T22:18:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T22:18:07,281 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:18:07,282 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:18:07,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:18:07,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731709087336, max timestamp=1731709087372Num uniques 1072; 2024-11-15T22:18:07,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731709087336, max timestamp=1731709087372Num uniques 1072; 2024-11-15T22:18:07,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-15T22:18:07,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-15T22:18:07,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731709087336, max timestamp=1731709087372Num uniques 1072; 2024-11-15T22:18:07,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-15T22:18:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T22:18:07,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:07,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:07,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731709087417, max timestamp=1731709087461Num uniques 1081; 2024-11-15T22:18:07,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-15T22:18:07,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731709087417, max timestamp=1731709087461Num uniques 1081; 2024-11-15T22:18:07,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-15T22:18:07,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731709087417, max timestamp=1731709087461Num uniques 1081; 2024-11-15T22:18:07,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-15T22:18:07,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731709087511, max timestamp=1731709087563Num uniques 1090; 2024-11-15T22:18:07,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-15T22:18:07,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731709087511, max timestamp=1731709087563Num uniques 1090; 2024-11-15T22:18:07,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-15T22:18:07,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731709087537, max timestamp=1731709087563Num uniques 1090; 2024-11-15T22:18:07,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-15T22:18:07,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:07,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:07,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:07,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:07,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:07,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:07,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/2b2a77fc2f704730903407520d015556 2024-11-15T22:18:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T22:18:07,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/0910b8602d524f278c96a209f44a3ddc is 50, key is test_row_0/C:col10/1731709086144/Put/seqid=0 2024-11-15T22:18:07,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742409_1585 (size=12151) 2024-11-15T22:18:07,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731709087636, max timestamp=1731709087735Num uniques 1099; 2024-11-15T22:18:07,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-15T22:18:07,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731709087636, max timestamp=1731709087735Num uniques 1099; 2024-11-15T22:18:07,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=6870, suffix heap size=142264,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=150 2024-11-15T22:18:07,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731709087636, max timestamp=1731709087735Num uniques 1049; 2024-11-15T22:18:07,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=6870, suffix heap size=137064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=150 2024-11-15T22:18:07,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:07,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:07,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731709087767, max timestamp=1731709087815Num uniques 1108; 2024-11-15T22:18:07,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-15T22:18:07,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1731709087767, max timestamp=1731709087815Num uniques 1009; 2024-11-15T22:18:07,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46204, new segment data size=9518, suffix heap size=132904,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=208 2024-11-15T22:18:07,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731709087767, max timestamp=1731709087816Num uniques 1009; 2024-11-15T22:18:07,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46206, new segment data size=9520, suffix heap size=132904,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=208 2024-11-15T22:18:07,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731709087860, max timestamp=1731709087898Num uniques 1067; 2024-11-15T22:18:07,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-15T22:18:07,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731709087860, max timestamp=1731709087898Num uniques 1067; 2024-11-15T22:18:07,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-15T22:18:07,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731709087860, max timestamp=1731709087898Num uniques 1067; 2024-11-15T22:18:07,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-15T22:18:07,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T22:18:07,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:07,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:07,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:07,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:07,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:07,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:07,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:07,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731709087954, max timestamp=1731709087967Num uniques 1026; 2024-11-15T22:18:07,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-15T22:18:07,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731709087954, max timestamp=1731709087967Num uniques 1026; 2024-11-15T22:18:07,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-15T22:18:07,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731709087954, max timestamp=1731709087967Num uniques 1026; 2024-11-15T22:18:07,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-15T22:18:08,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731709088006, max timestamp=1731709088046Num uniques 1085; 2024-11-15T22:18:08,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-15T22:18:08,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731709088006, max timestamp=1731709088046Num uniques 1085; 2024-11-15T22:18:08,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-15T22:18:08,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731709088006, max timestamp=1731709088046Num uniques 1085; 2024-11-15T22:18:08,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-15T22:18:08,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/0910b8602d524f278c96a209f44a3ddc 2024-11-15T22:18:08,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/e32c0a4e7457406b9f580965253d25c9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/e32c0a4e7457406b9f580965253d25c9 2024-11-15T22:18:08,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:08,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:08,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709148097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731709088066, max timestamp=1731709088106Num uniques 1094; 2024-11-15T22:18:08,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50094, new segment data size=6870, suffix heap size=141744,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=150 2024-11-15T22:18:08,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731709088066, max timestamp=1731709088106Num uniques 1094; 2024-11-15T22:18:08,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50094, new segment data size=6870, suffix heap size=141744,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=150 2024-11-15T22:18:08,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8874, totalHeapSize=20416, min timestamp=1731709088066, max timestamp=1731709088106Num uniques 1094; 2024-11-15T22:18:08,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50094, new segment data size=8874, suffix heap size=141744,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=194 2024-11-15T22:18:08,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/e32c0a4e7457406b9f580965253d25c9, entries=150, sequenceid=142, filesize=30.4 K 2024-11-15T22:18:08,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/2b2a77fc2f704730903407520d015556 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/2b2a77fc2f704730903407520d015556 2024-11-15T22:18:08,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/2b2a77fc2f704730903407520d015556, entries=150, sequenceid=142, filesize=11.9 K 2024-11-15T22:18:08,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/0910b8602d524f278c96a209f44a3ddc as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/0910b8602d524f278c96a209f44a3ddc 2024-11-15T22:18:08,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/0910b8602d524f278c96a209f44a3ddc, entries=150, sequenceid=142, filesize=11.9 K 2024-11-15T22:18:08,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=110.14 KB/112782 for e1d3c3e92f92bf8e015eabb60b759778 in 1680ms, sequenceid=142, compaction requested=true 2024-11-15T22:18:08,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:08,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1d3c3e92f92bf8e015eabb60b759778:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:18:08,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:18:08,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1d3c3e92f92bf8e015eabb60b759778:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:18:08,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:18:08,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1d3c3e92f92bf8e015eabb60b759778:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:18:08,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T22:18:08,167 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:18:08,167 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:18:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:08,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1d3c3e92f92bf8e015eabb60b759778 3/3 column families, dataSize=123.56 KB heapSize=352.84 KB 2024-11-15T22:18:08,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=A 2024-11-15T22:18:08,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:18:08,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=B 2024-11-15T22:18:08,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:18:08,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=C 2024-11-15T22:18:08,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:18:08,171 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:18:08,171 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): e1d3c3e92f92bf8e015eabb60b759778/C is initiating minor compaction (all files) 2024-11-15T22:18:08,172 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1d3c3e92f92bf8e015eabb60b759778/C in TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,172 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/188fcd7c59be4c6a8136f854c82d8899, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/adc49a21b5ee422eac145c41541c9f6e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/0910b8602d524f278c96a209f44a3ddc] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp, totalSize=33.3 K 2024-11-15T22:18:08,174 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:18:08,174 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): e1d3c3e92f92bf8e015eabb60b759778/A is initiating minor compaction (all files) 2024-11-15T22:18:08,174 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1d3c3e92f92bf8e015eabb60b759778/A in TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,174 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/00e265a869a2407fb346ee7a33abbed6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/3c7cdda8de594b85a0df1c19b2bc7282, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/e32c0a4e7457406b9f580965253d25c9] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp, totalSize=82.7 K 2024-11-15T22:18:08,174 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,174 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/00e265a869a2407fb346ee7a33abbed6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/3c7cdda8de594b85a0df1c19b2bc7282, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/e32c0a4e7457406b9f580965253d25c9] 2024-11-15T22:18:08,174 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 188fcd7c59be4c6a8136f854c82d8899, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731709083250 2024-11-15T22:18:08,192 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00e265a869a2407fb346ee7a33abbed6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731709083250 2024-11-15T22:18:08,192 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting adc49a21b5ee422eac145c41541c9f6e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731709083916 2024-11-15T22:18:08,193 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c7cdda8de594b85a0df1c19b2bc7282, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731709083916 2024-11-15T22:18:08,193 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 0910b8602d524f278c96a209f44a3ddc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731709086144 2024-11-15T22:18:08,196 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting e32c0a4e7457406b9f580965253d25c9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731709086144 2024-11-15T22:18:08,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cbff48b79ed74b96b9e05eb53538777a_e1d3c3e92f92bf8e015eabb60b759778 is 50, key is test_row_0/A:col10/1731709088165/Put/seqid=0 2024-11-15T22:18:08,221 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:08,225 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1d3c3e92f92bf8e015eabb60b759778#C#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:18:08,226 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/5cd2c4ed07544140ae225e16ee878d42 is 50, key is test_row_0/C:col10/1731709086144/Put/seqid=0 2024-11-15T22:18:08,228 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115c550b4afbee6483a8afe4b01934817e5_e1d3c3e92f92bf8e015eabb60b759778 store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:08,230 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115c550b4afbee6483a8afe4b01934817e5_e1d3c3e92f92bf8e015eabb60b759778, store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:08,231 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c550b4afbee6483a8afe4b01934817e5_e1d3c3e92f92bf8e015eabb60b759778 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:08,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709148223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709148229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709148229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709148229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709148231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742410_1586 (size=17534) 2024-11-15T22:18:08,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742412_1588 (size=4469) 2024-11-15T22:18:08,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742411_1587 (size=12493) 2024-11-15T22:18:08,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709148353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709148353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709148353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709148353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:08,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:08,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T22:18:08,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709148451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:08,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:08,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709148562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709148562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709148562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709148571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,693 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1d3c3e92f92bf8e015eabb60b759778#A#compaction#498 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:18:08,693 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/20de7563c930432f935f45d0e3fe2eba is 175, key is test_row_0/A:col10/1731709086144/Put/seqid=0 2024-11-15T22:18:08,696 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:08,720 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/5cd2c4ed07544140ae225e16ee878d42 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/5cd2c4ed07544140ae225e16ee878d42 2024-11-15T22:18:08,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:08,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:08,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,739 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cbff48b79ed74b96b9e05eb53538777a_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cbff48b79ed74b96b9e05eb53538777a_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:08,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742413_1589 (size=31447) 2024-11-15T22:18:08,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,749 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/4cd76f0a33e0470095aa6a7f47e38fc3, store: [table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:08,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/4cd76f0a33e0470095aa6a7f47e38fc3 is 175, key is test_row_0/A:col10/1731709088165/Put/seqid=0 2024-11-15T22:18:08,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709148772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,784 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1d3c3e92f92bf8e015eabb60b759778/C of e1d3c3e92f92bf8e015eabb60b759778 into 5cd2c4ed07544140ae225e16ee878d42(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:18:08,784 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:08,784 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., storeName=e1d3c3e92f92bf8e015eabb60b759778/C, priority=13, startTime=1731709088166; duration=0sec 2024-11-15T22:18:08,784 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:18:08,784 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1d3c3e92f92bf8e015eabb60b759778:C 2024-11-15T22:18:08,784 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:18:08,785 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:18:08,785 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): e1d3c3e92f92bf8e015eabb60b759778/B is initiating minor compaction (all files) 2024-11-15T22:18:08,785 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1d3c3e92f92bf8e015eabb60b759778/B in TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,786 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/11e205a18907412db2346a6260843b65, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/ad2cc05aac9b48bc92ad1a1b3eb2adba, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/2b2a77fc2f704730903407520d015556] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp, totalSize=33.3 K 2024-11-15T22:18:08,786 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 11e205a18907412db2346a6260843b65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731709083250 2024-11-15T22:18:08,786 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting ad2cc05aac9b48bc92ad1a1b3eb2adba, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731709083916 2024-11-15T22:18:08,786 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b2a77fc2f704730903407520d015556, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731709086144 2024-11-15T22:18:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742414_1590 (size=48639) 2024-11-15T22:18:08,798 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1d3c3e92f92bf8e015eabb60b759778#B#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:18:08,799 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/2ab83e6ae0b24e96b43255a2663d3fac is 50, key is test_row_0/B:col10/1731709086144/Put/seqid=0 2024-11-15T22:18:08,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742415_1591 (size=12493) 2024-11-15T22:18:08,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709148875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709148880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709148885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709148888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:08,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:08,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:08,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:08,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:08,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:08,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:09,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:09,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:09,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:09,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,180 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/20de7563c930432f935f45d0e3fe2eba as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/20de7563c930432f935f45d0e3fe2eba 2024-11-15T22:18:09,192 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=435, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/4cd76f0a33e0470095aa6a7f47e38fc3 2024-11-15T22:18:09,216 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1d3c3e92f92bf8e015eabb60b759778/A of e1d3c3e92f92bf8e015eabb60b759778 into 20de7563c930432f935f45d0e3fe2eba(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-15T22:18:09,216 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:09,216 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., storeName=e1d3c3e92f92bf8e015eabb60b759778/A, priority=13, startTime=1731709088166; duration=1sec 2024-11-15T22:18:09,216 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:18:09,217 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1d3c3e92f92bf8e015eabb60b759778:A 2024-11-15T22:18:09,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:09,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:09,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:09,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:09,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/3d5e3800f18e40208e8c39e64c23921b is 50, key is test_row_0/B:col10/1731709088165/Put/seqid=0 2024-11-15T22:18:09,280 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/2ab83e6ae0b24e96b43255a2663d3fac as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/2ab83e6ae0b24e96b43255a2663d3fac 2024-11-15T22:18:09,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709149301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:09,308 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1d3c3e92f92bf8e015eabb60b759778/B of e1d3c3e92f92bf8e015eabb60b759778 into 2ab83e6ae0b24e96b43255a2663d3fac(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:18:09,308 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:09,308 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., storeName=e1d3c3e92f92bf8e015eabb60b759778/B, priority=13, startTime=1731709088166; duration=0sec 2024-11-15T22:18:09,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:09,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:09,308 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:18:09,308 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1d3c3e92f92bf8e015eabb60b759778:B 2024-11-15T22:18:09,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:09,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742416_1592 (size=12301) 2024-11-15T22:18:09,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:09,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:09,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709149382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709149404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:09,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709149405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:09,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709149405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T22:18:09,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:09,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:09,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:09,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:09,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.53 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/3d5e3800f18e40208e8c39e64c23921b 2024-11-15T22:18:09,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:09,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:09,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:09,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:09,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/f2dcbcf6d5ad487cbb953ae4411b341d is 50, key is test_row_0/C:col10/1731709088165/Put/seqid=0 2024-11-15T22:18:09,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742417_1593 (size=12301) 2024-11-15T22:18:09,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:09,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:09,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:09,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:09,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:10,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:10,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:10,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:10,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.53 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/f2dcbcf6d5ad487cbb953ae4411b341d 2024-11-15T22:18:10,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:10,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:10,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:10,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:10,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/4cd76f0a33e0470095aa6a7f47e38fc3 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/4cd76f0a33e0470095aa6a7f47e38fc3 2024-11-15T22:18:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/4cd76f0a33e0470095aa6a7f47e38fc3, entries=250, sequenceid=435, filesize=47.5 K 2024-11-15T22:18:10,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/3d5e3800f18e40208e8c39e64c23921b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/3d5e3800f18e40208e8c39e64c23921b 2024-11-15T22:18:10,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/3d5e3800f18e40208e8c39e64c23921b, entries=150, sequenceid=435, filesize=12.0 K 2024-11-15T22:18:10,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/f2dcbcf6d5ad487cbb953ae4411b341d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/f2dcbcf6d5ad487cbb953ae4411b341d 2024-11-15T22:18:10,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/f2dcbcf6d5ad487cbb953ae4411b341d, entries=150, sequenceid=435, filesize=12.0 K 2024-11-15T22:18:10,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~123.56 KB/126522, heapSize ~352.80 KB/361264, currentSize=53.67 KB/54960 for e1d3c3e92f92bf8e015eabb60b759778 in 2098ms, sequenceid=435, compaction requested=false 2024-11-15T22:18:10,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:10,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1d3c3e92f92bf8e015eabb60b759778 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T22:18:10,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=A 2024-11-15T22:18:10,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:10,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=B 2024-11-15T22:18:10,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:10,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=C 2024-11-15T22:18:10,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:10,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:10,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:10,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:10,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:10,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ba9b147ad87f47db8cf6126a31bea334_e1d3c3e92f92bf8e015eabb60b759778 is 50, key is test_row_0/A:col10/1731709088219/Put/seqid=0 2024-11-15T22:18:10,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742418_1594 (size=12454) 2024-11-15T22:18:10,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709150462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:10,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709150468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:10,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709150468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:10,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:10,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709150472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:10,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709150473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:10,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:10,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:10,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:10,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:10,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709150584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:10,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709150584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:10,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709150591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:10,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709150592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:10,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709150605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:10,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:10,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:10,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:10,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:10,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709150801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:10,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:10,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709150801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:10,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709150811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:10,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709150812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:10,825 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:10,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:10,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:10,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:10,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:10,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709150825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:10,850 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ba9b147ad87f47db8cf6126a31bea334_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ba9b147ad87f47db8cf6126a31bea334_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:10,861 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/53b4df8b5ff84113a09b3963168dec31, store: [table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:10,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/53b4df8b5ff84113a09b3963168dec31 is 175, key is test_row_0/A:col10/1731709088219/Put/seqid=0 2024-11-15T22:18:10,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742419_1595 (size=31255) 2024-11-15T22:18:10,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:10,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:10,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:10,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:10,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709151115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:11,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709151122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:11,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709151124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:11,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709151124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:11,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:11,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:11,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:11,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:11,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:11,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:11,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709151145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:11,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:11,300 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=450, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/53b4df8b5ff84113a09b3963168dec31 2024-11-15T22:18:11,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:11,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:11,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:11,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:11,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/bc1f1f58c4ab4a42a5bf44c7b3cb921a is 50, key is test_row_0/B:col10/1731709088219/Put/seqid=0 2024-11-15T22:18:11,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742420_1596 (size=12301) 2024-11-15T22:18:11,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/bc1f1f58c4ab4a42a5bf44c7b3cb921a 2024-11-15T22:18:11,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/97f2a8a2e8fc48b99ec2a0a7a6940542 is 50, key is test_row_0/C:col10/1731709088219/Put/seqid=0 2024-11-15T22:18:11,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742421_1597 (size=12301) 2024-11-15T22:18:11,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:11,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:11,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:11,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:11,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T22:18:11,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:11,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:11,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:11,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:11,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709151632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:11,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709151632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:11,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:11,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:11,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709151632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:11,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709151641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:11,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:11,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709151664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:11,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:11,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:11,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:11,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:11,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:11,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:11,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:11,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/97f2a8a2e8fc48b99ec2a0a7a6940542 2024-11-15T22:18:11,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/53b4df8b5ff84113a09b3963168dec31 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/53b4df8b5ff84113a09b3963168dec31 2024-11-15T22:18:11,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/53b4df8b5ff84113a09b3963168dec31, entries=150, sequenceid=450, filesize=30.5 K 2024-11-15T22:18:11,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/bc1f1f58c4ab4a42a5bf44c7b3cb921a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/bc1f1f58c4ab4a42a5bf44c7b3cb921a 2024-11-15T22:18:11,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/bc1f1f58c4ab4a42a5bf44c7b3cb921a, entries=150, sequenceid=450, filesize=12.0 K 2024-11-15T22:18:11,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/97f2a8a2e8fc48b99ec2a0a7a6940542 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/97f2a8a2e8fc48b99ec2a0a7a6940542 2024-11-15T22:18:11,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/97f2a8a2e8fc48b99ec2a0a7a6940542, entries=150, sequenceid=450, filesize=12.0 K 2024-11-15T22:18:11,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e1d3c3e92f92bf8e015eabb60b759778 in 1571ms, sequenceid=450, compaction requested=true 2024-11-15T22:18:11,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:11,912 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:18:11,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1d3c3e92f92bf8e015eabb60b759778:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:18:11,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:18:11,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1d3c3e92f92bf8e015eabb60b759778:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:18:11,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:18:11,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1d3c3e92f92bf8e015eabb60b759778:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:18:11,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:18:11,913 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:18:11,914 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111341 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:18:11,914 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): e1d3c3e92f92bf8e015eabb60b759778/A is initiating minor compaction (all files) 2024-11-15T22:18:11,914 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1d3c3e92f92bf8e015eabb60b759778/A in TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:11,914 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/20de7563c930432f935f45d0e3fe2eba, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/4cd76f0a33e0470095aa6a7f47e38fc3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/53b4df8b5ff84113a09b3963168dec31] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp, totalSize=108.7 K 2024-11-15T22:18:11,914 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:11,914 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/20de7563c930432f935f45d0e3fe2eba, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/4cd76f0a33e0470095aa6a7f47e38fc3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/53b4df8b5ff84113a09b3963168dec31] 2024-11-15T22:18:11,914 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 20de7563c930432f935f45d0e3fe2eba, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731709086144 2024-11-15T22:18:11,915 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cd76f0a33e0470095aa6a7f47e38fc3, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731709088139 2024-11-15T22:18:11,916 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 53b4df8b5ff84113a09b3963168dec31, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731709088219 2024-11-15T22:18:11,916 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:18:11,916 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): e1d3c3e92f92bf8e015eabb60b759778/B is initiating minor compaction (all files) 2024-11-15T22:18:11,916 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1d3c3e92f92bf8e015eabb60b759778/B in TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:11,916 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/2ab83e6ae0b24e96b43255a2663d3fac, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/3d5e3800f18e40208e8c39e64c23921b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/bc1f1f58c4ab4a42a5bf44c7b3cb921a] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp, totalSize=36.2 K 2024-11-15T22:18:11,917 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ab83e6ae0b24e96b43255a2663d3fac, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731709086144 2024-11-15T22:18:11,917 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d5e3800f18e40208e8c39e64c23921b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731709088139 2024-11-15T22:18:11,918 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc1f1f58c4ab4a42a5bf44c7b3cb921a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731709088219 2024-11-15T22:18:11,932 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1d3c3e92f92bf8e015eabb60b759778#B#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:18:11,932 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/039f06663a914a07ac6337497fc506d0 is 50, key is test_row_0/B:col10/1731709088219/Put/seqid=0 2024-11-15T22:18:11,935 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:11,941 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115735b4af7f07d4da8a4f61c89280930a2_e1d3c3e92f92bf8e015eabb60b759778 store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:11,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T22:18:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:11,942 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing e1d3c3e92f92bf8e015eabb60b759778 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T22:18:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=A 2024-11-15T22:18:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=B 2024-11-15T22:18:11,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:11,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=C 2024-11-15T22:18:11,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:11,944 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115735b4af7f07d4da8a4f61c89280930a2_e1d3c3e92f92bf8e015eabb60b759778, store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:11,944 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115735b4af7f07d4da8a4f61c89280930a2_e1d3c3e92f92bf8e015eabb60b759778 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:11,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742422_1598 (size=12745) 2024-11-15T22:18:11,987 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/039f06663a914a07ac6337497fc506d0 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/039f06663a914a07ac6337497fc506d0 2024-11-15T22:18:11,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742423_1599 (size=4469) 2024-11-15T22:18:11,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153d4d50cd892848e2bf1269bbb2e3d14d_e1d3c3e92f92bf8e015eabb60b759778 is 50, key is test_row_0/A:col10/1731709090456/Put/seqid=0 2024-11-15T22:18:11,999 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1d3c3e92f92bf8e015eabb60b759778/B of e1d3c3e92f92bf8e015eabb60b759778 into 039f06663a914a07ac6337497fc506d0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:18:11,999 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:11,999 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., storeName=e1d3c3e92f92bf8e015eabb60b759778/B, priority=13, startTime=1731709091913; duration=0sec 2024-11-15T22:18:11,999 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:18:11,999 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1d3c3e92f92bf8e015eabb60b759778:B 2024-11-15T22:18:11,999 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:18:12,003 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:18:12,003 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): e1d3c3e92f92bf8e015eabb60b759778/C is initiating minor compaction (all files) 2024-11-15T22:18:12,003 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1d3c3e92f92bf8e015eabb60b759778/C in TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:12,003 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/5cd2c4ed07544140ae225e16ee878d42, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/f2dcbcf6d5ad487cbb953ae4411b341d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/97f2a8a2e8fc48b99ec2a0a7a6940542] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp, totalSize=36.2 K 2024-11-15T22:18:12,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742424_1600 (size=12454) 2024-11-15T22:18:12,004 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cd2c4ed07544140ae225e16ee878d42, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731709086144 2024-11-15T22:18:12,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,005 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2dcbcf6d5ad487cbb953ae4411b341d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731709088139 2024-11-15T22:18:12,005 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97f2a8a2e8fc48b99ec2a0a7a6940542, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731709088219 2024-11-15T22:18:12,017 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153d4d50cd892848e2bf1269bbb2e3d14d_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153d4d50cd892848e2bf1269bbb2e3d14d_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:12,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/6260339a2ec74899a8e205ac96f6e243, store: [table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:12,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/6260339a2ec74899a8e205ac96f6e243 is 175, key is test_row_0/A:col10/1731709090456/Put/seqid=0 2024-11-15T22:18:12,035 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1d3c3e92f92bf8e015eabb60b759778#C#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:18:12,036 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/f92f96e50c2549729ad827a6f5570dc3 is 50, key is test_row_0/C:col10/1731709088219/Put/seqid=0 2024-11-15T22:18:12,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742425_1601 (size=31255) 2024-11-15T22:18:12,091 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=470, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/6260339a2ec74899a8e205ac96f6e243 2024-11-15T22:18:12,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742426_1602 (size=12745) 2024-11-15T22:18:12,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/4e50e2cb78254669be4264a08967e01a is 50, key is test_row_0/B:col10/1731709090456/Put/seqid=0 2024-11-15T22:18:12,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742427_1603 (size=12301) 2024-11-15T22:18:12,149 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/4e50e2cb78254669be4264a08967e01a 2024-11-15T22:18:12,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/adba8fb5db1b453f93a1cc5098744b6c is 50, key is test_row_0/C:col10/1731709090456/Put/seqid=0 2024-11-15T22:18:12,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742428_1604 (size=12301) 2024-11-15T22:18:12,228 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/adba8fb5db1b453f93a1cc5098744b6c 2024-11-15T22:18:12,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/6260339a2ec74899a8e205ac96f6e243 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/6260339a2ec74899a8e205ac96f6e243 2024-11-15T22:18:12,275 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/6260339a2ec74899a8e205ac96f6e243, entries=150, sequenceid=470, filesize=30.5 K 2024-11-15T22:18:12,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/4e50e2cb78254669be4264a08967e01a as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/4e50e2cb78254669be4264a08967e01a 2024-11-15T22:18:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,284 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/4e50e2cb78254669be4264a08967e01a, entries=150, sequenceid=470, filesize=12.0 K 2024-11-15T22:18:12,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/adba8fb5db1b453f93a1cc5098744b6c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/adba8fb5db1b453f93a1cc5098744b6c 2024-11-15T22:18:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,294 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/adba8fb5db1b453f93a1cc5098744b6c, entries=150, sequenceid=470, filesize=12.0 K 2024-11-15T22:18:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,294 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for e1d3c3e92f92bf8e015eabb60b759778 in 352ms, sequenceid=470, compaction requested=false 2024-11-15T22:18:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:12,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:12,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-15T22:18:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-15T22:18:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-15T22:18:12,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.0160 sec 2024-11-15T22:18:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 5.0370 sec 2024-11-15T22:18:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,396 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1d3c3e92f92bf8e015eabb60b759778#A#compaction#507 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:18:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,397 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/c20efe2a3a964c899d6271e607be1d52 is 175, key is test_row_0/A:col10/1731709088219/Put/seqid=0 2024-11-15T22:18:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742429_1605 (size=31699) 2024-11-15T22:18:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,423 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/c20efe2a3a964c899d6271e607be1d52 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/c20efe2a3a964c899d6271e607be1d52 2024-11-15T22:18:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,436 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1d3c3e92f92bf8e015eabb60b759778/A of e1d3c3e92f92bf8e015eabb60b759778 into c20efe2a3a964c899d6271e607be1d52(size=31.0 K), total size for store is 61.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:18:12,437 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:12,437 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., storeName=e1d3c3e92f92bf8e015eabb60b759778/A, priority=13, startTime=1731709091912; duration=0sec 2024-11-15T22:18:12,437 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:18:12,437 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1d3c3e92f92bf8e015eabb60b759778:A 2024-11-15T22:18:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,528 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/f92f96e50c2549729ad827a6f5570dc3 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/f92f96e50c2549729ad827a6f5570dc3 2024-11-15T22:18:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,548 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1d3c3e92f92bf8e015eabb60b759778/C of e1d3c3e92f92bf8e015eabb60b759778 into f92f96e50c2549729ad827a6f5570dc3(size=12.4 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:18:12,548 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:12,549 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., storeName=e1d3c3e92f92bf8e015eabb60b759778/C, priority=13, startTime=1731709091913; duration=0sec 2024-11-15T22:18:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,549 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:18:12,549 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1d3c3e92f92bf8e015eabb60b759778:C 2024-11-15T22:18:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:12,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1d3c3e92f92bf8e015eabb60b759778 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T22:18:12,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=A 2024-11-15T22:18:12,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:12,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=B 2024-11-15T22:18:12,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:12,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=C 2024-11-15T22:18:12,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111537b34e70bbb94331baa9ea332c86a14b_e1d3c3e92f92bf8e015eabb60b759778 is 50, key is test_row_0/A:col10/1731709092700/Put/seqid=0 2024-11-15T22:18:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:12,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742430_1606 (size=20074) 2024-11-15T22:18:12,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731709092746, max timestamp=1731709092771Num uniques 859; 2024-11-15T22:18:12,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731709092746, max timestamp=1731709092771Num uniques 859; 2024-11-15T22:18:12,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:18:12,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T22:18:12,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731709092761, max timestamp=1731709092771Num uniques 859; 2024-11-15T22:18:12,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:18:12,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709152817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:12,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709152823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:12,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:12,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709152864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:12,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:12,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709152885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:12,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:12,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709152889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:12,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709152992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:12,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709152992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:12,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:12,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:12,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:12,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:12,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:12,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:13,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709152995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:13,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:13,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709153001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709153001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:13,150 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:13,156 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111537b34e70bbb94331baa9ea332c86a14b_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111537b34e70bbb94331baa9ea332c86a14b_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:13,172 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/053cc7e2939645808dfb238e431e82a5, store: [table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:13,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/053cc7e2939645808dfb238e431e82a5 is 175, key is test_row_0/A:col10/1731709092700/Put/seqid=0 2024-11-15T22:18:13,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742431_1607 (size=57333) 2024-11-15T22:18:13,205 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=483, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/053cc7e2939645808dfb238e431e82a5 2024-11-15T22:18:13,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:13,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709153202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:13,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:13,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709153212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:13,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709153212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:13,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:13,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:13,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709153212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:13,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:13,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709153212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:13,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/19da8e3c72f04fac952a118eeb23aa05 is 50, key is test_row_0/B:col10/1731709092700/Put/seqid=0 2024-11-15T22:18:13,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742432_1608 (size=12301) 2024-11-15T22:18:13,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709153521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:13,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:13,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709153522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:13,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:13,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709153532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709153532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:13,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709153532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:13,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/19da8e3c72f04fac952a118eeb23aa05 2024-11-15T22:18:13,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/2aa01217eb204a48b653727352484d0c is 50, key is test_row_0/C:col10/1731709092700/Put/seqid=0 2024-11-15T22:18:13,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742433_1609 (size=12301) 2024-11-15T22:18:14,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709154031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:14,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:14,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709154042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:14,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709154042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:14,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709154052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:14,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709154054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:14,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:14,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:14,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/2aa01217eb204a48b653727352484d0c 2024-11-15T22:18:14,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/053cc7e2939645808dfb238e431e82a5 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/053cc7e2939645808dfb238e431e82a5 2024-11-15T22:18:14,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/053cc7e2939645808dfb238e431e82a5, entries=300, sequenceid=483, filesize=56.0 K 2024-11-15T22:18:14,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/19da8e3c72f04fac952a118eeb23aa05 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/19da8e3c72f04fac952a118eeb23aa05 2024-11-15T22:18:14,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/19da8e3c72f04fac952a118eeb23aa05, entries=150, sequenceid=483, filesize=12.0 K 2024-11-15T22:18:14,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/2aa01217eb204a48b653727352484d0c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/2aa01217eb204a48b653727352484d0c 2024-11-15T22:18:14,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/2aa01217eb204a48b653727352484d0c, entries=150, sequenceid=483, filesize=12.0 K 2024-11-15T22:18:14,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.65 KB/141980 for e1d3c3e92f92bf8e015eabb60b759778 in 1529ms, sequenceid=483, compaction requested=true 2024-11-15T22:18:14,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:14,231 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:18:14,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1d3c3e92f92bf8e015eabb60b759778:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:18:14,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:18:14,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1d3c3e92f92bf8e015eabb60b759778:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:18:14,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:18:14,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1d3c3e92f92bf8e015eabb60b759778:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:18:14,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:18:14,232 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:18:14,232 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1541): e1d3c3e92f92bf8e015eabb60b759778/A is initiating minor compaction (all files) 2024-11-15T22:18:14,232 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1d3c3e92f92bf8e015eabb60b759778/A in TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:14,232 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:18:14,232 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/c20efe2a3a964c899d6271e607be1d52, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/6260339a2ec74899a8e205ac96f6e243, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/053cc7e2939645808dfb238e431e82a5] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp, totalSize=117.5 K 2024-11-15T22:18:14,232 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:14,232 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. files: [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/c20efe2a3a964c899d6271e607be1d52, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/6260339a2ec74899a8e205ac96f6e243, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/053cc7e2939645808dfb238e431e82a5] 2024-11-15T22:18:14,236 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting c20efe2a3a964c899d6271e607be1d52, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731709088219 2024-11-15T22:18:14,240 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 6260339a2ec74899a8e205ac96f6e243, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731709090456 2024-11-15T22:18:14,241 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:18:14,241 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): e1d3c3e92f92bf8e015eabb60b759778/B is initiating minor compaction (all files) 2024-11-15T22:18:14,241 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1d3c3e92f92bf8e015eabb60b759778/B in TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:14,241 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/039f06663a914a07ac6337497fc506d0, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/4e50e2cb78254669be4264a08967e01a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/19da8e3c72f04fac952a118eeb23aa05] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp, totalSize=36.5 K 2024-11-15T22:18:14,241 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] compactions.Compactor(225): Compacting 053cc7e2939645808dfb238e431e82a5, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731709092684 2024-11-15T22:18:14,243 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 039f06663a914a07ac6337497fc506d0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731709088219 2024-11-15T22:18:14,244 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e50e2cb78254669be4264a08967e01a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731709090456 2024-11-15T22:18:14,244 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19da8e3c72f04fac952a118eeb23aa05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731709092696 2024-11-15T22:18:14,256 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:14,262 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1d3c3e92f92bf8e015eabb60b759778#B#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:18:14,263 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/3d147c9cfe8049bdaeed7e6b419613d8 is 50, key is test_row_0/B:col10/1731709092700/Put/seqid=0 2024-11-15T22:18:14,265 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115c317d9314f6043f3b490003f7b380b3e_e1d3c3e92f92bf8e015eabb60b759778 store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:14,267 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115c317d9314f6043f3b490003f7b380b3e_e1d3c3e92f92bf8e015eabb60b759778, store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:14,267 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c317d9314f6043f3b490003f7b380b3e_e1d3c3e92f92bf8e015eabb60b759778 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:14,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742434_1610 (size=12847) 2024-11-15T22:18:14,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742435_1611 (size=4469) 2024-11-15T22:18:14,324 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/3d147c9cfe8049bdaeed7e6b419613d8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/3d147c9cfe8049bdaeed7e6b419613d8 2024-11-15T22:18:14,325 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1d3c3e92f92bf8e015eabb60b759778#A#compaction#515 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:18:14,326 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/abecb3c66f2846d0b073e57a1b377fb3 is 175, key is test_row_0/A:col10/1731709092700/Put/seqid=0 2024-11-15T22:18:14,330 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1d3c3e92f92bf8e015eabb60b759778/B of e1d3c3e92f92bf8e015eabb60b759778 into 3d147c9cfe8049bdaeed7e6b419613d8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:18:14,330 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:14,330 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., storeName=e1d3c3e92f92bf8e015eabb60b759778/B, priority=13, startTime=1731709094231; duration=0sec 2024-11-15T22:18:14,331 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:18:14,331 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1d3c3e92f92bf8e015eabb60b759778:B 2024-11-15T22:18:14,331 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:18:14,332 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:18:14,332 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1541): e1d3c3e92f92bf8e015eabb60b759778/C is initiating minor compaction (all files) 2024-11-15T22:18:14,332 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1d3c3e92f92bf8e015eabb60b759778/C in TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:14,332 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/f92f96e50c2549729ad827a6f5570dc3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/adba8fb5db1b453f93a1cc5098744b6c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/2aa01217eb204a48b653727352484d0c] into tmpdir=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp, totalSize=36.5 K 2024-11-15T22:18:14,339 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting f92f96e50c2549729ad827a6f5570dc3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731709088219 2024-11-15T22:18:14,339 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting adba8fb5db1b453f93a1cc5098744b6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731709090456 2024-11-15T22:18:14,339 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aa01217eb204a48b653727352484d0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731709092696 2024-11-15T22:18:14,359 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1d3c3e92f92bf8e015eabb60b759778#C#compaction#517 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:18:14,360 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/08030cf9873a49b3ab150edcebc2665c is 50, key is test_row_0/C:col10/1731709092700/Put/seqid=0 2024-11-15T22:18:14,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742436_1612 (size=31801) 2024-11-15T22:18:14,374 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/abecb3c66f2846d0b073e57a1b377fb3 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/abecb3c66f2846d0b073e57a1b377fb3 2024-11-15T22:18:14,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742437_1613 (size=12847) 2024-11-15T22:18:14,420 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1d3c3e92f92bf8e015eabb60b759778/A of e1d3c3e92f92bf8e015eabb60b759778 into abecb3c66f2846d0b073e57a1b377fb3(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:18:14,420 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:14,420 INFO [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., storeName=e1d3c3e92f92bf8e015eabb60b759778/A, priority=13, startTime=1731709094231; duration=0sec 2024-11-15T22:18:14,420 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:18:14,420 DEBUG [RS:0;e611192d6313:38779-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1d3c3e92f92bf8e015eabb60b759778:A 2024-11-15T22:18:14,446 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/08030cf9873a49b3ab150edcebc2665c as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/08030cf9873a49b3ab150edcebc2665c 2024-11-15T22:18:14,460 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1d3c3e92f92bf8e015eabb60b759778/C of e1d3c3e92f92bf8e015eabb60b759778 into 08030cf9873a49b3ab150edcebc2665c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:18:14,460 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:14,460 INFO [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., storeName=e1d3c3e92f92bf8e015eabb60b759778/C, priority=13, startTime=1731709094232; duration=0sec 2024-11-15T22:18:14,461 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:18:14,461 DEBUG [RS:0;e611192d6313:38779-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1d3c3e92f92bf8e015eabb60b759778:C 2024-11-15T22:18:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:15,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731709092815, max timestamp=1731709095067Num uniques 1068; 2024-11-15T22:18:15,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-15T22:18:15,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731709092815, max timestamp=1731709095067Num uniques 1068; 2024-11-15T22:18:15,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-15T22:18:15,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731709092815, max timestamp=1731709095068Num uniques 1018; 2024-11-15T22:18:15,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-15T22:18:15,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1d3c3e92f92bf8e015eabb60b759778 3/3 column families, dataSize=39.46 KB heapSize=103.41 KB 2024-11-15T22:18:15,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=A 2024-11-15T22:18:15,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:18:15,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=B 2024-11-15T22:18:15,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:18:15,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=C 2024-11-15T22:18:15,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:18:15,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111505b57acd556441eb85fa1823fb7a76fb_e1d3c3e92f92bf8e015eabb60b759778 is 50, key is test_row_0/A:col10/1731709092822/Put/seqid=0 2024-11-15T22:18:15,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731709095118, max timestamp=1731709095130Num uniques 859; 2024-11-15T22:18:15,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:18:15,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731709095118, max timestamp=1731709095130Num uniques 859; 2024-11-15T22:18:15,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:18:15,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731709095118, max timestamp=1731709095130Num uniques 859; 2024-11-15T22:18:15,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T22:18:15,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709155180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709155181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731709095171, max timestamp=1731709095183Num uniques 1018; 2024-11-15T22:18:15,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-15T22:18:15,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731709095171, max timestamp=1731709095183Num uniques 1018; 2024-11-15T22:18:15,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-15T22:18:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731709095171, max timestamp=1731709095183Num uniques 1018; 2024-11-15T22:18:15,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-15T22:18:15,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731709095223, max timestamp=1731709095242Num uniques 1009; 2024-11-15T22:18:15,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-11-15T22:18:15,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731709095223, max timestamp=1731709095242Num uniques 1009; 2024-11-15T22:18:15,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-11-15T22:18:15,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731709095223, max timestamp=1731709095242Num uniques 1009; 2024-11-15T22:18:15,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-11-15T22:18:15,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731709095299, max timestamp=1731709095315Num uniques 1036; 2024-11-15T22:18:15,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-15T22:18:15,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731709095299, max timestamp=1731709095315Num uniques 1036; 2024-11-15T22:18:15,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-11-15T22:18:15,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731709095299, max timestamp=1731709095315Num uniques 1036; 2024-11-15T22:18:15,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-15T22:18:15,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731709095344, max timestamp=1731709095369Num uniques 1045; 2024-11-15T22:18:15,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-15T22:18:15,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731709095344, max timestamp=1731709095369Num uniques 1045; 2024-11-15T22:18:15,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731709095344, max timestamp=1731709095369Num uniques 1009; 2024-11-15T22:18:15,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13510, suffix heap size=132904,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=295 2024-11-15T22:18:15,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e611192d6313:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-15T22:18:15,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709155449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709155450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709155454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709155462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709155469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T22:18:15,512 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:18:15,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:18:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-15T22:18:15,514 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:18:15,515 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:18:15,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:18:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T22:18:15,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709155572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709155572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709155572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709155581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709155582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T22:18:15,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T22:18:15,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:15,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:15,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:15,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:15,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:15,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709155782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709155784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709155792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709155792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709155803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:15,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T22:18:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:15,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T22:18:15,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742438_1614 (size=12454) 2024-11-15T22:18:15,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T22:18:15,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:15,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:15,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:15,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:15,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:15,992 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,003 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111505b57acd556441eb85fa1823fb7a76fb_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111505b57acd556441eb85fa1823fb7a76fb_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:16,012 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/f984912674be4fcaaa3f432cd76e439f, store: [table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:16,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/f984912674be4fcaaa3f432cd76e439f is 175, key is test_row_0/A:col10/1731709092822/Put/seqid=0 2024-11-15T22:18:16,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742439_1615 (size=31255) 2024-11-15T22:18:16,026 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=526, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/f984912674be4fcaaa3f432cd76e439f 2024-11-15T22:18:16,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/24f71a5b3882400894bf1b4e48d6c788 is 50, key is test_row_0/B:col10/1731709092822/Put/seqid=0 2024-11-15T22:18:16,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709156101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:16,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742440_1616 (size=12301) 2024-11-15T22:18:16,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709156102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:16,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709156102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:16,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709156104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:16,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709156113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T22:18:16,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T22:18:16,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:16,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:16,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:16,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T22:18:16,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:16,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:16,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:16,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T22:18:16,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:16,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:16,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:16,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/24f71a5b3882400894bf1b4e48d6c788 2024-11-15T22:18:16,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/35e2effdb5734081ab66b19253c07257 is 50, key is test_row_0/C:col10/1731709092822/Put/seqid=0 2024-11-15T22:18:16,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742441_1617 (size=12301) 2024-11-15T22:18:16,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43224 deadline: 1731709156613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:16,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1731709156625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:16,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43272 deadline: 1731709156625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:16,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43198 deadline: 1731709156625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:16,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709156625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:16,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:16,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:16,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:16,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T22:18:16,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:16,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:16,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:16,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T22:18:16,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T22:18:16,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:16,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:16,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:16,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,812 INFO [Thread-2439 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2439. 2024-11-15T22:18:16,812 DEBUG [Thread-2439 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:18:16,812 DEBUG [Thread-2439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:16,812 DEBUG [Thread-2439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:16,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:18:16,814 INFO [Thread-2441 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2441. 2024-11-15T22:18:16,814 DEBUG [Thread-2441 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:18:16,814 DEBUG [Thread-2441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:16,814 DEBUG [Thread-2441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:16,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:18:16,819 INFO [Thread-2445 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2445. 2024-11-15T22:18:16,819 INFO [Thread-2443 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2443. 2024-11-15T22:18:16,820 DEBUG [Thread-2443 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:18:16,820 DEBUG [Thread-2443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:16,820 DEBUG [Thread-2443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:16,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:18:16,820 DEBUG [Thread-2445 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:18:16,820 DEBUG [Thread-2445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:16,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:18:16,821 DEBUG [Thread-2445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:16,823 INFO [Thread-2437 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2437. 2024-11-15T22:18:16,823 DEBUG [Thread-2437 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:18:16,823 DEBUG [Thread-2437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:16,823 DEBUG [Thread-2437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:16,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:18:16,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T22:18:16,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:16,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:16,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:16,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:18:16,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.62 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/35e2effdb5734081ab66b19253c07257 2024-11-15T22:18:16,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/f984912674be4fcaaa3f432cd76e439f as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/f984912674be4fcaaa3f432cd76e439f 2024-11-15T22:18:17,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/f984912674be4fcaaa3f432cd76e439f, entries=150, sequenceid=526, filesize=30.5 K 2024-11-15T22:18:17,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/24f71a5b3882400894bf1b4e48d6c788 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/24f71a5b3882400894bf1b4e48d6c788 2024-11-15T22:18:17,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/24f71a5b3882400894bf1b4e48d6c788, entries=150, sequenceid=526, filesize=12.0 K 2024-11-15T22:18:17,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/35e2effdb5734081ab66b19253c07257 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/35e2effdb5734081ab66b19253c07257 2024-11-15T22:18:17,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/35e2effdb5734081ab66b19253c07257, entries=150, sequenceid=526, filesize=12.0 K 2024-11-15T22:18:17,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.46 KB/40402, heapSize ~103.36 KB/105840, currentSize=154.31 KB/158010 for e1d3c3e92f92bf8e015eabb60b759778 in 1958ms, sequenceid=526, compaction requested=false 2024-11-15T22:18:17,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:17,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T22:18:17,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:17,111 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing e1d3c3e92f92bf8e015eabb60b759778 3/3 column families, dataSize=154.31 KB heapSize=432 KB 2024-11-15T22:18:17,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=A 2024-11-15T22:18:17,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:18:17,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=B 2024-11-15T22:18:17,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:18:17,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=C 2024-11-15T22:18:17,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T22:18:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159ed7f151aa774564a33a4dda1826e81d_e1d3c3e92f92bf8e015eabb60b759778 is 50, key is test_row_0/A:col10/1731709095460/Put/seqid=0 2024-11-15T22:18:17,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742442_1618 (size=12454) 2024-11-15T22:18:17,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:17,563 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159ed7f151aa774564a33a4dda1826e81d_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159ed7f151aa774564a33a4dda1826e81d_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:17,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/9545a11b6b8843ef9fbd9d1a0fc0109d, store: [table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:17,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/9545a11b6b8843ef9fbd9d1a0fc0109d is 175, key is test_row_0/A:col10/1731709095460/Put/seqid=0 2024-11-15T22:18:17,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742443_1619 (size=31255) 2024-11-15T22:18:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(8855): Flush requested on e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:17,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. as already flushing 2024-11-15T22:18:17,634 INFO [Thread-2430 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2430. 2024-11-15T22:18:17,634 DEBUG [Thread-2430 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:18:17,634 DEBUG [Thread-2430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:17,634 DEBUG [Thread-2430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:17,634 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:18:17,645 INFO [Thread-2426 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2426. 2024-11-15T22:18:17,645 INFO [Thread-2432 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2432. 2024-11-15T22:18:17,645 DEBUG [Thread-2426 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:18:17,645 DEBUG [Thread-2432 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:18:17,645 DEBUG [Thread-2426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:17,645 DEBUG [Thread-2432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:17,645 DEBUG [Thread-2426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:17,645 DEBUG [Thread-2432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:17,645 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:18:17,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:18:17,650 INFO [Thread-2434 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2434. 2024-11-15T22:18:17,650 DEBUG [Thread-2434 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:18:17,650 DEBUG [Thread-2434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:17,650 DEBUG [Thread-2434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:17,650 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:18:17,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:18:17,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38779 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43194 deadline: 1731709157652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 2024-11-15T22:18:17,653 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e611192d6313,38779,1731708927034 for 'test_row_0' in e1d3c3e92f92bf8e015eabb60b759778 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T22:18:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 , the old value is region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:18:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., hostname=e611192d6313,38779,1731708927034, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:18:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T22:18:17,974 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=632, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/9545a11b6b8843ef9fbd9d1a0fc0109d 2024-11-15T22:18:17,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/95d303065ed74615a8c6a9487f2411b9 is 50, key is test_row_0/B:col10/1731709095460/Put/seqid=0 2024-11-15T22:18:17,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742444_1620 (size=12301) 2024-11-15T22:18:18,402 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/95d303065ed74615a8c6a9487f2411b9 2024-11-15T22:18:18,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/71659d45a5e846938ccae948d270da4d is 50, key is test_row_0/C:col10/1731709095460/Put/seqid=0 2024-11-15T22:18:18,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742445_1621 (size=12301) 2024-11-15T22:18:18,818 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/71659d45a5e846938ccae948d270da4d 2024-11-15T22:18:18,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/9545a11b6b8843ef9fbd9d1a0fc0109d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/9545a11b6b8843ef9fbd9d1a0fc0109d 2024-11-15T22:18:18,823 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/9545a11b6b8843ef9fbd9d1a0fc0109d, entries=150, sequenceid=632, filesize=30.5 K 2024-11-15T22:18:18,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/95d303065ed74615a8c6a9487f2411b9 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/95d303065ed74615a8c6a9487f2411b9 2024-11-15T22:18:18,826 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/95d303065ed74615a8c6a9487f2411b9, entries=150, sequenceid=632, filesize=12.0 K 2024-11-15T22:18:18,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/71659d45a5e846938ccae948d270da4d as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/71659d45a5e846938ccae948d270da4d 2024-11-15T22:18:18,830 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/71659d45a5e846938ccae948d270da4d, entries=150, sequenceid=632, filesize=12.0 K 2024-11-15T22:18:18,831 INFO [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.95 KB/442320, currentSize=26.84 KB/27480 for e1d3c3e92f92bf8e015eabb60b759778 in 1720ms, sequenceid=632, compaction requested=true 2024-11-15T22:18:18,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for e1d3c3e92f92bf8e015eabb60b759778: 2024-11-15T22:18:18,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:18,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-15T22:18:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-15T22:18:18,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-15T22:18:18,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3180 sec 2024-11-15T22:18:18,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 3.3230 sec 2024-11-15T22:18:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T22:18:19,672 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:18:19,673 INFO [Thread-2428 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2428. 2024-11-15T22:18:19,673 DEBUG [Thread-2428 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:18:19,673 DEBUG [Thread-2428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:19,673 DEBUG [Thread-2428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:19,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:18:19,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T22:18:19,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-15T22:18:19,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 173 2024-11-15T22:18:19,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 173 2024-11-15T22:18:19,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-15T22:18:19,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-15T22:18:19,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T22:18:19,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2127 2024-11-15T22:18:19,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2123 2024-11-15T22:18:19,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2080 2024-11-15T22:18:19,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2147 2024-11-15T22:18:19,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2116 2024-11-15T22:18:19,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T22:18:19,676 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:18:19,676 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:18:19,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:19,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:19,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:18:19,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T22:18:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T22:18:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T22:18:19,680 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731709099680"}]},"ts":"1731709099680"} 2024-11-15T22:18:19,681 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T22:18:19,681 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T22:18:19,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:18:19,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1d3c3e92f92bf8e015eabb60b759778, UNASSIGN}] 2024-11-15T22:18:19,683 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1d3c3e92f92bf8e015eabb60b759778, UNASSIGN 2024-11-15T22:18:19,684 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=152 updating hbase:meta row=e1d3c3e92f92bf8e015eabb60b759778, regionState=CLOSING, regionLocation=e611192d6313,38779,1731708927034 2024-11-15T22:18:19,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1d3c3e92f92bf8e015eabb60b759778, UNASSIGN because future has completed 2024-11-15T22:18:19,686 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:18:19,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; CloseRegionProcedure e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034}] 2024-11-15T22:18:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T22:18:19,837 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(122): Close e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:19,838 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:18:19,838 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1722): Closing e1d3c3e92f92bf8e015eabb60b759778, disabling compactions & flushes 2024-11-15T22:18:19,838 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:19,838 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:19,838 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. after waiting 0 ms 2024-11-15T22:18:19,838 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:19,838 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(2902): Flushing e1d3c3e92f92bf8e015eabb60b759778 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T22:18:19,838 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=A 2024-11-15T22:18:19,838 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:19,838 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=B 2024-11-15T22:18:19,838 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:19,838 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1d3c3e92f92bf8e015eabb60b759778, store=C 2024-11-15T22:18:19,838 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T22:18:19,843 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150d9acd7b165c404f9de651d6e16614b0_e1d3c3e92f92bf8e015eabb60b759778 is 50, key is test_row_0/A:col10/1731709099672/Put/seqid=0 2024-11-15T22:18:19,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742446_1622 (size=9914) 2024-11-15T22:18:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T22:18:20,248 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:18:20,252 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150d9acd7b165c404f9de651d6e16614b0_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d9acd7b165c404f9de651d6e16614b0_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:20,253 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/52e2ceaf211e4ac18006815aa3b42497, store: [table=TestAcidGuarantees family=A region=e1d3c3e92f92bf8e015eabb60b759778] 2024-11-15T22:18:20,254 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/52e2ceaf211e4ac18006815aa3b42497 is 175, key is test_row_0/A:col10/1731709099672/Put/seqid=0 2024-11-15T22:18:20,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742447_1623 (size=22561) 2024-11-15T22:18:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T22:18:20,659 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=640, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/52e2ceaf211e4ac18006815aa3b42497 2024-11-15T22:18:20,674 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/f86670a9878243009f387d0f6a0b8cf0 is 50, key is test_row_0/B:col10/1731709099672/Put/seqid=0 2024-11-15T22:18:20,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742448_1624 (size=9857) 2024-11-15T22:18:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T22:18:21,092 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/f86670a9878243009f387d0f6a0b8cf0 2024-11-15T22:18:21,105 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/1b1e02b68c044731b5b23bec75a1e0d8 is 50, key is test_row_0/C:col10/1731709099672/Put/seqid=0 2024-11-15T22:18:21,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742449_1625 (size=9857) 2024-11-15T22:18:21,529 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/1b1e02b68c044731b5b23bec75a1e0d8 2024-11-15T22:18:21,543 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/A/52e2ceaf211e4ac18006815aa3b42497 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/52e2ceaf211e4ac18006815aa3b42497 2024-11-15T22:18:21,562 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/52e2ceaf211e4ac18006815aa3b42497, entries=100, sequenceid=640, filesize=22.0 K 2024-11-15T22:18:21,563 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/B/f86670a9878243009f387d0f6a0b8cf0 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/f86670a9878243009f387d0f6a0b8cf0 2024-11-15T22:18:21,577 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/f86670a9878243009f387d0f6a0b8cf0, entries=100, sequenceid=640, filesize=9.6 K 2024-11-15T22:18:21,579 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/.tmp/C/1b1e02b68c044731b5b23bec75a1e0d8 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/1b1e02b68c044731b5b23bec75a1e0d8 2024-11-15T22:18:21,584 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/1b1e02b68c044731b5b23bec75a1e0d8, entries=100, sequenceid=640, filesize=9.6 K 2024-11-15T22:18:21,585 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e1d3c3e92f92bf8e015eabb60b759778 in 1747ms, sequenceid=640, compaction requested=true 2024-11-15T22:18:21,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/2f8d01aefb1141bdbd349f94ec3a0105, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/557982411faa429fb542b768ac625156, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/8d75eb1d9b6241d3a74539a297ee7c76, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/568ea4a27a894e1888b63ac93e9f4c6e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/1d28b5cec2d64411b6d6d0de816d1a06, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/9c54bc69e0d4465c948a8dcabf7745c7, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/6bab8206cf8440ffba76dcb144a80c2b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/00e265a869a2407fb346ee7a33abbed6, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/3c7cdda8de594b85a0df1c19b2bc7282, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/20de7563c930432f935f45d0e3fe2eba, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/e32c0a4e7457406b9f580965253d25c9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/4cd76f0a33e0470095aa6a7f47e38fc3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/c20efe2a3a964c899d6271e607be1d52, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/53b4df8b5ff84113a09b3963168dec31, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/6260339a2ec74899a8e205ac96f6e243, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/053cc7e2939645808dfb238e431e82a5] to archive 2024-11-15T22:18:21,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:18:21,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/2f8d01aefb1141bdbd349f94ec3a0105 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/2f8d01aefb1141bdbd349f94ec3a0105 2024-11-15T22:18:21,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/557982411faa429fb542b768ac625156 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/557982411faa429fb542b768ac625156 2024-11-15T22:18:21,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/8d75eb1d9b6241d3a74539a297ee7c76 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/8d75eb1d9b6241d3a74539a297ee7c76 2024-11-15T22:18:21,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/568ea4a27a894e1888b63ac93e9f4c6e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/568ea4a27a894e1888b63ac93e9f4c6e 2024-11-15T22:18:21,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/1d28b5cec2d64411b6d6d0de816d1a06 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/1d28b5cec2d64411b6d6d0de816d1a06 2024-11-15T22:18:21,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/9c54bc69e0d4465c948a8dcabf7745c7 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/9c54bc69e0d4465c948a8dcabf7745c7 2024-11-15T22:18:21,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/6bab8206cf8440ffba76dcb144a80c2b to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/6bab8206cf8440ffba76dcb144a80c2b 2024-11-15T22:18:21,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/00e265a869a2407fb346ee7a33abbed6 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/00e265a869a2407fb346ee7a33abbed6 2024-11-15T22:18:21,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/3c7cdda8de594b85a0df1c19b2bc7282 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/3c7cdda8de594b85a0df1c19b2bc7282 2024-11-15T22:18:21,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/20de7563c930432f935f45d0e3fe2eba to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/20de7563c930432f935f45d0e3fe2eba 2024-11-15T22:18:21,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/e32c0a4e7457406b9f580965253d25c9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/e32c0a4e7457406b9f580965253d25c9 2024-11-15T22:18:21,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/4cd76f0a33e0470095aa6a7f47e38fc3 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/4cd76f0a33e0470095aa6a7f47e38fc3 2024-11-15T22:18:21,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/c20efe2a3a964c899d6271e607be1d52 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/c20efe2a3a964c899d6271e607be1d52 2024-11-15T22:18:21,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/53b4df8b5ff84113a09b3963168dec31 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/53b4df8b5ff84113a09b3963168dec31 2024-11-15T22:18:21,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/6260339a2ec74899a8e205ac96f6e243 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/6260339a2ec74899a8e205ac96f6e243 2024-11-15T22:18:21,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/053cc7e2939645808dfb238e431e82a5 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/053cc7e2939645808dfb238e431e82a5 2024-11-15T22:18:21,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/0b81c3981ab44da99dafd3b2387cc154, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/1f0cb8b1e4e74d9ab9447da4ab0ad66d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/12949ba19be446c78921e7adf2333109, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/532786a2c612437abdd3e5ed81e3176b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/37365b47305c4f57accec674165bb04d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/4ca7c30c865047b0880d5913267eafe9, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/11e205a18907412db2346a6260843b65, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/79f3ef3bf5194d9d88ce89dc303ac6ba, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/ad2cc05aac9b48bc92ad1a1b3eb2adba, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/2ab83e6ae0b24e96b43255a2663d3fac, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/2b2a77fc2f704730903407520d015556, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/3d5e3800f18e40208e8c39e64c23921b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/039f06663a914a07ac6337497fc506d0, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/bc1f1f58c4ab4a42a5bf44c7b3cb921a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/4e50e2cb78254669be4264a08967e01a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/19da8e3c72f04fac952a118eeb23aa05] to archive 2024-11-15T22:18:21,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:18:21,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/0b81c3981ab44da99dafd3b2387cc154 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/0b81c3981ab44da99dafd3b2387cc154 2024-11-15T22:18:21,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/1f0cb8b1e4e74d9ab9447da4ab0ad66d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/1f0cb8b1e4e74d9ab9447da4ab0ad66d 2024-11-15T22:18:21,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/12949ba19be446c78921e7adf2333109 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/12949ba19be446c78921e7adf2333109 2024-11-15T22:18:21,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/532786a2c612437abdd3e5ed81e3176b to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/532786a2c612437abdd3e5ed81e3176b 2024-11-15T22:18:21,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/37365b47305c4f57accec674165bb04d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/37365b47305c4f57accec674165bb04d 2024-11-15T22:18:21,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/4ca7c30c865047b0880d5913267eafe9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/4ca7c30c865047b0880d5913267eafe9 2024-11-15T22:18:21,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/11e205a18907412db2346a6260843b65 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/11e205a18907412db2346a6260843b65 2024-11-15T22:18:21,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/79f3ef3bf5194d9d88ce89dc303ac6ba to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/79f3ef3bf5194d9d88ce89dc303ac6ba 2024-11-15T22:18:21,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/ad2cc05aac9b48bc92ad1a1b3eb2adba to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/ad2cc05aac9b48bc92ad1a1b3eb2adba 2024-11-15T22:18:21,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/2ab83e6ae0b24e96b43255a2663d3fac to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/2ab83e6ae0b24e96b43255a2663d3fac 2024-11-15T22:18:21,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/2b2a77fc2f704730903407520d015556 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/2b2a77fc2f704730903407520d015556 2024-11-15T22:18:21,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/3d5e3800f18e40208e8c39e64c23921b to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/3d5e3800f18e40208e8c39e64c23921b 2024-11-15T22:18:21,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/039f06663a914a07ac6337497fc506d0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/039f06663a914a07ac6337497fc506d0 2024-11-15T22:18:21,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/bc1f1f58c4ab4a42a5bf44c7b3cb921a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/bc1f1f58c4ab4a42a5bf44c7b3cb921a 2024-11-15T22:18:21,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/4e50e2cb78254669be4264a08967e01a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/4e50e2cb78254669be4264a08967e01a 2024-11-15T22:18:21,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/19da8e3c72f04fac952a118eeb23aa05 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/19da8e3c72f04fac952a118eeb23aa05 2024-11-15T22:18:21,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/67776852e377407d9fe2d6ac36afb53d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/bb4998b3187d42abb53f795bb8c5148a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/d5fd084958b2454398e9a17758e6b95a, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/a9b327162090461faf101e5079ac374b, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/b1a32ffd93ad4782a5c5da8ea7d32a70, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/ed64efa3335545ce8f58de7a9db12e93, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/188fcd7c59be4c6a8136f854c82d8899, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/73775e49fc974727b57324e571220378, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/adc49a21b5ee422eac145c41541c9f6e, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/5cd2c4ed07544140ae225e16ee878d42, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/0910b8602d524f278c96a209f44a3ddc, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/f2dcbcf6d5ad487cbb953ae4411b341d, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/f92f96e50c2549729ad827a6f5570dc3, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/97f2a8a2e8fc48b99ec2a0a7a6940542, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/adba8fb5db1b453f93a1cc5098744b6c, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/2aa01217eb204a48b653727352484d0c] to archive 2024-11-15T22:18:21,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:18:21,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/67776852e377407d9fe2d6ac36afb53d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/67776852e377407d9fe2d6ac36afb53d 2024-11-15T22:18:21,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/bb4998b3187d42abb53f795bb8c5148a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/bb4998b3187d42abb53f795bb8c5148a 2024-11-15T22:18:21,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/d5fd084958b2454398e9a17758e6b95a to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/d5fd084958b2454398e9a17758e6b95a 2024-11-15T22:18:21,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/a9b327162090461faf101e5079ac374b to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/a9b327162090461faf101e5079ac374b 2024-11-15T22:18:21,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/b1a32ffd93ad4782a5c5da8ea7d32a70 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/b1a32ffd93ad4782a5c5da8ea7d32a70 2024-11-15T22:18:21,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/ed64efa3335545ce8f58de7a9db12e93 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/ed64efa3335545ce8f58de7a9db12e93 2024-11-15T22:18:21,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/188fcd7c59be4c6a8136f854c82d8899 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/188fcd7c59be4c6a8136f854c82d8899 2024-11-15T22:18:21,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/73775e49fc974727b57324e571220378 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/73775e49fc974727b57324e571220378 2024-11-15T22:18:21,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/adc49a21b5ee422eac145c41541c9f6e to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/adc49a21b5ee422eac145c41541c9f6e 2024-11-15T22:18:21,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/5cd2c4ed07544140ae225e16ee878d42 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/5cd2c4ed07544140ae225e16ee878d42 2024-11-15T22:18:21,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/0910b8602d524f278c96a209f44a3ddc to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/0910b8602d524f278c96a209f44a3ddc 2024-11-15T22:18:21,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/f2dcbcf6d5ad487cbb953ae4411b341d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/f2dcbcf6d5ad487cbb953ae4411b341d 2024-11-15T22:18:21,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/f92f96e50c2549729ad827a6f5570dc3 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/f92f96e50c2549729ad827a6f5570dc3 2024-11-15T22:18:21,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/97f2a8a2e8fc48b99ec2a0a7a6940542 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/97f2a8a2e8fc48b99ec2a0a7a6940542 2024-11-15T22:18:21,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/adba8fb5db1b453f93a1cc5098744b6c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/adba8fb5db1b453f93a1cc5098744b6c 2024-11-15T22:18:21,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/2aa01217eb204a48b653727352484d0c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/2aa01217eb204a48b653727352484d0c 2024-11-15T22:18:21,781 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/recovered.edits/643.seqid, newMaxSeqId=643, maxSeqId=4 2024-11-15T22:18:21,793 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. 2024-11-15T22:18:21,793 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1676): Region close journal for e1d3c3e92f92bf8e015eabb60b759778: Waiting for close lock at 1731709099838Running coprocessor pre-close hooks at 1731709099838Disabling compacts and flushes for region at 1731709099838Disabling writes for close at 1731709099838Obtaining lock to block concurrent updates at 1731709099838Preparing flush snapshotting stores in e1d3c3e92f92bf8e015eabb60b759778 at 1731709099838Finished memstore snapshotting TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731709099838Flushing stores of TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778. at 1731709099838Flushing e1d3c3e92f92bf8e015eabb60b759778/A: creating writer at 1731709099838Flushing mob file e1d3c3e92f92bf8e015eabb60b759778/A: appending metadata at 1731709099843 (+5 ms)Flushing mob file e1d3c3e92f92bf8e015eabb60b759778/A: closing flushed file at 1731709099843Flushing e1d3c3e92f92bf8e015eabb60b759778/A: appending metadata at 1731709100253 (+410 ms)Flushing e1d3c3e92f92bf8e015eabb60b759778/A: closing flushed file at 1731709100253Flushing e1d3c3e92f92bf8e015eabb60b759778/B: creating writer at 1731709100662 (+409 ms)Flushing e1d3c3e92f92bf8e015eabb60b759778/B: appending metadata at 1731709100674 (+12 ms)Flushing e1d3c3e92f92bf8e015eabb60b759778/B: closing flushed file at 1731709100674Flushing e1d3c3e92f92bf8e015eabb60b759778/C: creating writer at 1731709101101 (+427 ms)Flushing e1d3c3e92f92bf8e015eabb60b759778/C: appending metadata at 1731709101105 (+4 ms)Flushing e1d3c3e92f92bf8e015eabb60b759778/C: closing flushed file at 1731709101105Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74c17a99: reopening flushed file at 1731709101536 (+431 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e0693ae: reopening flushed file at 1731709101562 (+26 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14f562b: reopening flushed file at 1731709101577 (+15 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e1d3c3e92f92bf8e015eabb60b759778 in 1747ms, sequenceid=640, compaction requested=true at 1731709101585 (+8 ms)Writing region close event to WAL at 1731709101761 (+176 ms)Running coprocessor post-close hooks at 1731709101793 (+32 ms)Closed at 1731709101793 2024-11-15T22:18:21,800 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(157): Closed e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,801 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=152 updating hbase:meta row=e1d3c3e92f92bf8e015eabb60b759778, regionState=CLOSED 2024-11-15T22:18:21,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE, hasLock=false; CloseRegionProcedure e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 because future has completed 2024-11-15T22:18:21,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-15T22:18:21,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; CloseRegionProcedure e1d3c3e92f92bf8e015eabb60b759778, server=e611192d6313,38779,1731708927034 in 2.1230 sec 2024-11-15T22:18:21,824 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-15T22:18:21,824 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1d3c3e92f92bf8e015eabb60b759778, UNASSIGN in 2.1340 sec 2024-11-15T22:18:21,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-15T22:18:21,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1430 sec 2024-11-15T22:18:21,828 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731709101828"}]},"ts":"1731709101828"} 2024-11-15T22:18:21,830 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T22:18:21,830 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T22:18:21,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1540 sec 2024-11-15T22:18:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T22:18:21,852 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:18:21,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T22:18:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:18:21,855 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=154, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:18:21,856 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=154, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:18:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T22:18:21,858 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,860 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C, FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/recovered.edits] 2024-11-15T22:18:21,873 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/52e2ceaf211e4ac18006815aa3b42497 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/52e2ceaf211e4ac18006815aa3b42497 2024-11-15T22:18:21,877 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/9545a11b6b8843ef9fbd9d1a0fc0109d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/9545a11b6b8843ef9fbd9d1a0fc0109d 2024-11-15T22:18:21,881 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/abecb3c66f2846d0b073e57a1b377fb3 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/abecb3c66f2846d0b073e57a1b377fb3 2024-11-15T22:18:21,884 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/f984912674be4fcaaa3f432cd76e439f to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/A/f984912674be4fcaaa3f432cd76e439f 2024-11-15T22:18:21,894 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/24f71a5b3882400894bf1b4e48d6c788 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/24f71a5b3882400894bf1b4e48d6c788 2024-11-15T22:18:21,896 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/3d147c9cfe8049bdaeed7e6b419613d8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/3d147c9cfe8049bdaeed7e6b419613d8 2024-11-15T22:18:21,897 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/95d303065ed74615a8c6a9487f2411b9 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/95d303065ed74615a8c6a9487f2411b9 2024-11-15T22:18:21,898 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/f86670a9878243009f387d0f6a0b8cf0 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/B/f86670a9878243009f387d0f6a0b8cf0 2024-11-15T22:18:21,900 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/08030cf9873a49b3ab150edcebc2665c to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/08030cf9873a49b3ab150edcebc2665c 2024-11-15T22:18:21,901 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/1b1e02b68c044731b5b23bec75a1e0d8 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/1b1e02b68c044731b5b23bec75a1e0d8 2024-11-15T22:18:21,902 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/35e2effdb5734081ab66b19253c07257 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/35e2effdb5734081ab66b19253c07257 2024-11-15T22:18:21,902 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/71659d45a5e846938ccae948d270da4d to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/C/71659d45a5e846938ccae948d270da4d 2024-11-15T22:18:21,904 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/recovered.edits/643.seqid to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778/recovered.edits/643.seqid 2024-11-15T22:18:21,905 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/default/TestAcidGuarantees/e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,905 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T22:18:21,905 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T22:18:21,906 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T22:18:21,908 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111505b57acd556441eb85fa1823fb7a76fb_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111505b57acd556441eb85fa1823fb7a76fb_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,912 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d9acd7b165c404f9de651d6e16614b0_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d9acd7b165c404f9de651d6e16614b0_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,913 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152c4f3a5c1f684d5f87208b68f1163d3b_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152c4f3a5c1f684d5f87208b68f1163d3b_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,914 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111537b34e70bbb94331baa9ea332c86a14b_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111537b34e70bbb94331baa9ea332c86a14b_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,916 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153d4d50cd892848e2bf1269bbb2e3d14d_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153d4d50cd892848e2bf1269bbb2e3d14d_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,917 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111541a28a6a1f014aa19868cb2ed3a612bd_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111541a28a6a1f014aa19868cb2ed3a612bd_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,918 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115765a07a6ccae48bab23815298bd4b2d3_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115765a07a6ccae48bab23815298bd4b2d3_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,918 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158862d7d942204301952b3a4a25a7a198_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158862d7d942204301952b3a4a25a7a198_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,919 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111592f4ca0cecb749e4a4e8530dddecd0ef_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111592f4ca0cecb749e4a4e8530dddecd0ef_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,920 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159ed7f151aa774564a33a4dda1826e81d_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159ed7f151aa774564a33a4dda1826e81d_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,920 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a25100afa37a4ebf9d3fae46d6fcfdf9_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a25100afa37a4ebf9d3fae46d6fcfdf9_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,921 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a39339f121a94adc99519b3eef6edf9c_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a39339f121a94adc99519b3eef6edf9c_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,922 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b64815c2bf9f48ff8f78117211a0b5cb_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b64815c2bf9f48ff8f78117211a0b5cb_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,923 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ba9b147ad87f47db8cf6126a31bea334_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ba9b147ad87f47db8cf6126a31bea334_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,923 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cbff48b79ed74b96b9e05eb53538777a_e1d3c3e92f92bf8e015eabb60b759778 to hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cbff48b79ed74b96b9e05eb53538777a_e1d3c3e92f92bf8e015eabb60b759778 2024-11-15T22:18:21,924 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T22:18:21,928 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=154, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:18:21,930 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T22:18:21,932 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T22:18:21,934 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=154, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:18:21,934 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T22:18:21,934 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731709101934"}]},"ts":"9223372036854775807"} 2024-11-15T22:18:21,936 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T22:18:21,936 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e1d3c3e92f92bf8e015eabb60b759778, NAME => 'TestAcidGuarantees,,1731709073590.e1d3c3e92f92bf8e015eabb60b759778.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T22:18:21,936 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T22:18:21,937 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731709101936"}]},"ts":"9223372036854775807"} 2024-11-15T22:18:21,938 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T22:18:21,939 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=154, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:18:21,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 86 msec 2024-11-15T22:18:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T22:18:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T22:18:21,962 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:18:21,974 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252 (was 248) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1247 (was 1291), ProcessCount=11 (was 11), AvailableMemoryMB=230 (was 648) 2024-11-15T22:18:21,974 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T22:18:21,974 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:18:21,974 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:18:21,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:21,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:21,975 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:18:21,976 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T22:18:21,976 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2117780854, stopped=false 2024-11-15T22:18:21,977 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e611192d6313,34969,1731708925722 2024-11-15T22:18:22,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T22:18:22,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:18:22,061 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T22:18:22,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T22:18:22,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:18:22,064 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:18:22,064 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:18:22,064 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:18:22,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:22,065 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e611192d6313,38779,1731708927034' ***** 2024-11-15T22:18:22,065 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T22:18:22,065 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T22:18:22,066 INFO [RS:0;e611192d6313:38779 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T22:18:22,067 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T22:18:22,067 INFO [RS:0;e611192d6313:38779 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T22:18:22,067 INFO [RS:0;e611192d6313:38779 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T22:18:22,068 INFO [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(959): stopping server e611192d6313,38779,1731708927034 2024-11-15T22:18:22,068 INFO [RS:0;e611192d6313:38779 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T22:18:22,068 INFO [RS:0;e611192d6313:38779 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e611192d6313:38779. 2024-11-15T22:18:22,068 DEBUG [RS:0;e611192d6313:38779 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:18:22,068 DEBUG [RS:0;e611192d6313:38779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:22,069 INFO [RS:0;e611192d6313:38779 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T22:18:22,069 INFO [RS:0;e611192d6313:38779 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T22:18:22,069 INFO [RS:0;e611192d6313:38779 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T22:18:22,070 INFO [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T22:18:22,074 INFO [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T22:18:22,075 DEBUG [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-15T22:18:22,075 DEBUG [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T22:18:22,076 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T22:18:22,077 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T22:18:22,077 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T22:18:22,077 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T22:18:22,077 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T22:18:22,077 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-15T22:18:22,117 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/.tmp/info/ff853847542848738096e46ebb7962e0 is 95, key is TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9./info:/1731708959215/DeleteFamily/seqid=0 2024-11-15T22:18:22,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742450_1626 (size=6514) 2024-11-15T22:18:22,132 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/.tmp/info/ff853847542848738096e46ebb7962e0 2024-11-15T22:18:22,144 INFO [regionserver/e611192d6313:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T22:18:22,156 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/.tmp/ns/e17d67443986454e903e3a6173764947 is 93, key is TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9./ns:/1731708959167/DeleteFamily/seqid=0 2024-11-15T22:18:22,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742451_1627 (size=5906) 2024-11-15T22:18:22,162 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/.tmp/ns/e17d67443986454e903e3a6173764947 2024-11-15T22:18:22,191 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/.tmp/rep_barrier/541849003d794ee3a5da25e4e38fb4b2 is 102, key is TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9./rep_barrier:/1731708959167/DeleteFamily/seqid=0 2024-11-15T22:18:22,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742452_1628 (size=6025) 2024-11-15T22:18:22,233 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/.tmp/rep_barrier/541849003d794ee3a5da25e4e38fb4b2 2024-11-15T22:18:22,269 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/.tmp/table/d360d9cd242f44f591bb986aceb7b072 is 96, key is TestAcidGuarantees,,1731708931418.0cf44e7263c9528c718ed800ce4e85e9./table:/1731708959167/DeleteFamily/seqid=0 2024-11-15T22:18:22,275 DEBUG [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T22:18:22,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742453_1629 (size=5936) 2024-11-15T22:18:22,294 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/.tmp/table/d360d9cd242f44f591bb986aceb7b072 2024-11-15T22:18:22,300 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/.tmp/info/ff853847542848738096e46ebb7962e0 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/info/ff853847542848738096e46ebb7962e0 2024-11-15T22:18:22,311 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/info/ff853847542848738096e46ebb7962e0, entries=12, sequenceid=90, filesize=6.4 K 2024-11-15T22:18:22,312 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/.tmp/ns/e17d67443986454e903e3a6173764947 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/ns/e17d67443986454e903e3a6173764947 2024-11-15T22:18:22,324 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/ns/e17d67443986454e903e3a6173764947, entries=8, sequenceid=90, filesize=5.8 K 2024-11-15T22:18:22,333 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/.tmp/rep_barrier/541849003d794ee3a5da25e4e38fb4b2 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/rep_barrier/541849003d794ee3a5da25e4e38fb4b2 2024-11-15T22:18:22,343 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/rep_barrier/541849003d794ee3a5da25e4e38fb4b2, entries=6, sequenceid=90, filesize=5.9 K 2024-11-15T22:18:22,348 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/.tmp/table/d360d9cd242f44f591bb986aceb7b072 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/table/d360d9cd242f44f591bb986aceb7b072 2024-11-15T22:18:22,364 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/table/d360d9cd242f44f591bb986aceb7b072, entries=7, sequenceid=90, filesize=5.8 K 2024-11-15T22:18:22,373 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 296ms, sequenceid=90, compaction requested=false 2024-11-15T22:18:22,417 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-15T22:18:22,421 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T22:18:22,421 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T22:18:22,421 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731709102076Running coprocessor pre-close hooks at 1731709102076Disabling compacts and flushes for region at 1731709102076Disabling writes for close at 1731709102077 (+1 ms)Obtaining lock to block concurrent updates at 1731709102077Preparing flush snapshotting stores in 1588230740 at 1731709102077Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731709102077Flushing stores of hbase:meta,,1.1588230740 at 1731709102084 (+7 ms)Flushing 1588230740/info: creating writer at 1731709102084Flushing 1588230740/info: appending metadata at 1731709102116 (+32 ms)Flushing 1588230740/info: closing flushed file at 1731709102117 (+1 ms)Flushing 1588230740/ns: creating writer at 1731709102137 (+20 ms)Flushing 1588230740/ns: appending metadata at 1731709102156 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731709102156Flushing 1588230740/rep_barrier: creating writer at 1731709102168 (+12 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731709102190 (+22 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731709102191 (+1 ms)Flushing 1588230740/table: creating writer at 1731709102239 (+48 ms)Flushing 1588230740/table: appending metadata at 1731709102269 (+30 ms)Flushing 1588230740/table: closing flushed file at 1731709102269Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a5320dc: reopening flushed file at 1731709102299 (+30 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@202dd487: reopening flushed file at 1731709102311 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51ab0871: reopening flushed file at 1731709102325 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3316b5a9: reopening flushed file at 1731709102343 (+18 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 296ms, sequenceid=90, compaction requested=false at 1731709102373 (+30 ms)Writing region close event to WAL at 1731709102406 (+33 ms)Running coprocessor post-close hooks at 1731709102420 (+14 ms)Closed at 1731709102421 (+1 ms) 2024-11-15T22:18:22,421 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T22:18:22,476 INFO [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(976): stopping server e611192d6313,38779,1731708927034; all regions closed. 2024-11-15T22:18:22,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741834_1010 (size=25344) 2024-11-15T22:18:22,497 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/WALs/e611192d6313,38779,1731708927034/e611192d6313%2C38779%2C1731708927034.meta.1731708930906.meta not finished, retry = 0 2024-11-15T22:18:22,600 DEBUG [RS:0;e611192d6313:38779 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/oldWALs 2024-11-15T22:18:22,600 INFO [RS:0;e611192d6313:38779 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e611192d6313%2C38779%2C1731708927034.meta:.meta(num 1731708930906) 2024-11-15T22:18:22,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741832_1008 (size=20035393) 2024-11-15T22:18:22,603 DEBUG [RS:0;e611192d6313:38779 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/oldWALs 2024-11-15T22:18:22,603 INFO [RS:0;e611192d6313:38779 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e611192d6313%2C38779%2C1731708927034:(num 1731708929896) 2024-11-15T22:18:22,603 DEBUG [RS:0;e611192d6313:38779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:18:22,603 INFO [RS:0;e611192d6313:38779 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T22:18:22,603 INFO [RS:0;e611192d6313:38779 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T22:18:22,603 INFO [RS:0;e611192d6313:38779 {}] hbase.ChoreService(370): Chore service for: regionserver/e611192d6313:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T22:18:22,604 INFO [RS:0;e611192d6313:38779 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T22:18:22,604 INFO [RS:0;e611192d6313:38779 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38779 2024-11-15T22:18:22,606 INFO [regionserver/e611192d6313:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T22:18:22,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e611192d6313,38779,1731708927034 2024-11-15T22:18:22,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T22:18:22,627 INFO [RS:0;e611192d6313:38779 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T22:18:22,629 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e611192d6313,38779,1731708927034] 2024-11-15T22:18:22,648 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e611192d6313,38779,1731708927034 already deleted, retry=false 2024-11-15T22:18:22,648 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e611192d6313,38779,1731708927034 expired; onlineServers=0 2024-11-15T22:18:22,649 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e611192d6313,34969,1731708925722' ***** 2024-11-15T22:18:22,649 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T22:18:22,649 INFO [M:0;e611192d6313:34969 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T22:18:22,649 INFO [M:0;e611192d6313:34969 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T22:18:22,649 DEBUG [M:0;e611192d6313:34969 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T22:18:22,649 DEBUG [M:0;e611192d6313:34969 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T22:18:22,649 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T22:18:22,649 DEBUG [master/e611192d6313:0:becomeActiveMaster-HFileCleaner.large.0-1731708929592 {}] cleaner.HFileCleaner(306): Exit Thread[master/e611192d6313:0:becomeActiveMaster-HFileCleaner.large.0-1731708929592,5,FailOnTimeoutGroup] 2024-11-15T22:18:22,649 INFO [M:0;e611192d6313:34969 {}] hbase.ChoreService(370): Chore service for: master/e611192d6313:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T22:18:22,649 INFO [M:0;e611192d6313:34969 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T22:18:22,649 DEBUG [M:0;e611192d6313:34969 {}] master.HMaster(1795): Stopping service threads 2024-11-15T22:18:22,650 INFO [M:0;e611192d6313:34969 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T22:18:22,650 INFO [M:0;e611192d6313:34969 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:18:22,650 ERROR [M:0;e611192d6313:34969 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1087708870) connection to localhost/127.0.0.1:36935 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:36935,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-15T22:18:22,650 INFO [M:0;e611192d6313:34969 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:18:22,650 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T22:18:22,650 DEBUG [master/e611192d6313:0:becomeActiveMaster-HFileCleaner.small.0-1731708929608 {}] cleaner.HFileCleaner(306): Exit Thread[master/e611192d6313:0:becomeActiveMaster-HFileCleaner.small.0-1731708929608,5,FailOnTimeoutGroup] 2024-11-15T22:18:22,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T22:18:22,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:18:22,669 DEBUG [M:0;e611192d6313:34969 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-15T22:18:22,669 DEBUG [M:0;e611192d6313:34969 {}] master.ActiveMasterManager(353): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-15T22:18:22,670 INFO [M:0;e611192d6313:34969 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/.lastflushedseqids 2024-11-15T22:18:22,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742454_1630 (size=72) 2024-11-15T22:18:22,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T22:18:22,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10140912f520001, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T22:18:22,738 INFO [RS:0;e611192d6313:38779 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T22:18:22,739 INFO [RS:0;e611192d6313:38779 {}] regionserver.HRegionServer(1031): Exiting; stopping=e611192d6313,38779,1731708927034; zookeeper connection closed. 2024-11-15T22:18:22,739 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5efdd944 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5efdd944 2024-11-15T22:18:22,739 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-15T22:18:23,081 INFO [M:0;e611192d6313:34969 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:18:23,082 INFO [M:0;e611192d6313:34969 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T22:18:23,082 DEBUG [M:0;e611192d6313:34969 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T22:18:23,082 INFO [M:0;e611192d6313:34969 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:18:23,082 DEBUG [M:0;e611192d6313:34969 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:18:23,082 DEBUG [M:0;e611192d6313:34969 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T22:18:23,082 DEBUG [M:0;e611192d6313:34969 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:18:23,082 INFO [M:0;e611192d6313:34969 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=752.16 KB heapSize=919.77 KB 2024-11-15T22:18:23,101 DEBUG [M:0;e611192d6313:34969 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1464e16bcb34f45a0fee6b2767de552 is 82, key is hbase:meta,,1/info:regioninfo/1731708930987/Put/seqid=0 2024-11-15T22:18:23,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742455_1631 (size=5672) 2024-11-15T22:18:23,536 INFO [M:0;e611192d6313:34969 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2076 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1464e16bcb34f45a0fee6b2767de552 2024-11-15T22:18:23,584 DEBUG [M:0;e611192d6313:34969 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b8e59635aee44ea697eb1e717c98fe5b is 2480, key is \x00\x00\x00\x00\x00\x00\x00R/proc:d/1731709014451/Put/seqid=0 2024-11-15T22:18:23,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742456_1632 (size=42632) 2024-11-15T22:18:23,599 INFO [M:0;e611192d6313:34969 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=751.61 KB at sequenceid=2076 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b8e59635aee44ea697eb1e717c98fe5b 2024-11-15T22:18:23,619 INFO [M:0;e611192d6313:34969 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b8e59635aee44ea697eb1e717c98fe5b 2024-11-15T22:18:23,657 DEBUG [M:0;e611192d6313:34969 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/79a8a16d583240668ec8452d69279810 is 69, key is e611192d6313,38779,1731708927034/rs:state/1731708929648/Put/seqid=0 2024-11-15T22:18:23,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073742457_1633 (size=5156) 2024-11-15T22:18:24,078 INFO [M:0;e611192d6313:34969 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2076 (bloomFilter=true), to=hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/79a8a16d583240668ec8452d69279810 2024-11-15T22:18:24,083 DEBUG [M:0;e611192d6313:34969 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1464e16bcb34f45a0fee6b2767de552 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b1464e16bcb34f45a0fee6b2767de552 2024-11-15T22:18:24,088 INFO [M:0;e611192d6313:34969 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b1464e16bcb34f45a0fee6b2767de552, entries=8, sequenceid=2076, filesize=5.5 K 2024-11-15T22:18:24,088 DEBUG [M:0;e611192d6313:34969 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b8e59635aee44ea697eb1e717c98fe5b as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b8e59635aee44ea697eb1e717c98fe5b 2024-11-15T22:18:24,092 INFO [M:0;e611192d6313:34969 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b8e59635aee44ea697eb1e717c98fe5b 2024-11-15T22:18:24,092 INFO [M:0;e611192d6313:34969 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b8e59635aee44ea697eb1e717c98fe5b, entries=154, sequenceid=2076, filesize=41.6 K 2024-11-15T22:18:24,093 DEBUG [M:0;e611192d6313:34969 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/79a8a16d583240668ec8452d69279810 as hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/79a8a16d583240668ec8452d69279810 2024-11-15T22:18:24,114 INFO [M:0;e611192d6313:34969 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36935/user/jenkins/test-data/9b1af586-0537-afba-ff15-b05c0c15b53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/79a8a16d583240668ec8452d69279810, entries=1, sequenceid=2076, filesize=5.0 K 2024-11-15T22:18:24,115 INFO [M:0;e611192d6313:34969 {}] regionserver.HRegion(3140): Finished flush of dataSize ~752.16 KB/770215, heapSize ~919.48 KB/941544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1033ms, sequenceid=2076, compaction requested=false 2024-11-15T22:18:24,121 INFO [M:0;e611192d6313:34969 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:18:24,121 DEBUG [M:0;e611192d6313:34969 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731709103082Disabling compacts and flushes for region at 1731709103082Disabling writes for close at 1731709103082Obtaining lock to block concurrent updates at 1731709103082Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731709103082Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=770215, getHeapSize=941784, getOffHeapSize=0, getCellsCount=2255 at 1731709103083 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731709103083Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731709103083Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731709103101 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731709103101Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731709103545 (+444 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731709103583 (+38 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731709103583Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731709103619 (+36 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731709103657 (+38 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731709103657Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31f6a0d3: reopening flushed file at 1731709104082 (+425 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43ddc10b: reopening flushed file at 1731709104088 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15ec5be4: reopening flushed file at 1731709104092 (+4 ms)Finished flush of dataSize ~752.16 KB/770215, heapSize ~919.48 KB/941544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1033ms, sequenceid=2076, compaction requested=false at 1731709104115 (+23 ms)Writing region close event to WAL at 1731709104121 (+6 ms)Closed at 1731709104121 2024-11-15T22:18:24,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38455 is added to blk_1073741830_1006 (size=906290) 2024-11-15T22:18:24,126 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T22:18:24,126 INFO [M:0;e611192d6313:34969 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T22:18:24,126 INFO [M:0;e611192d6313:34969 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34969 2024-11-15T22:18:24,126 INFO [M:0;e611192d6313:34969 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T22:18:24,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T22:18:24,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34969-0x10140912f520000, quorum=127.0.0.1:51187, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T22:18:24,305 INFO [M:0;e611192d6313:34969 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T22:18:24,315 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d1d0948{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T22:18:24,317 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@49b49063{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T22:18:24,318 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T22:18:24,318 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58f8c8ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T22:18:24,318 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6aa6ac55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/hadoop.log.dir/,STOPPED} 2024-11-15T22:18:24,326 WARN [BP-14505792-172.17.0.3-1731708921465 heartbeating to localhost/127.0.0.1:36935 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T22:18:24,326 WARN [BP-14505792-172.17.0.3-1731708921465 heartbeating to localhost/127.0.0.1:36935 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-14505792-172.17.0.3-1731708921465 (Datanode Uuid 2f720da5-324f-4b5d-90f7-54311f9c5ee1) service to localhost/127.0.0.1:36935 2024-11-15T22:18:24,327 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T22:18:24,327 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T22:18:24,329 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/cluster_a6235007-a423-ea83-cf16-0fd92ac68712/data/data1/current/BP-14505792-172.17.0.3-1731708921465 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T22:18:24,329 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/cluster_a6235007-a423-ea83-cf16-0fd92ac68712/data/data2/current/BP-14505792-172.17.0.3-1731708921465 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T22:18:24,330 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T22:18:24,337 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T22:18:24,340 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T22:18:24,340 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T22:18:24,340 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T22:18:24,341 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7bfe3323-50d4-2db2-6c39-c2da96ff750e/hadoop.log.dir/,STOPPED} 2024-11-15T22:18:24,356 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T22:18:24,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down